Browse Source

Merge remote-tracking branch 'upstream/master' into edge-3.3-locale-updates

pull/4918/head
Volodymyr Babak 5 years ago
parent
commit
080299c782
  1. 40
      application/src/main/data/json/demo/dashboards/firmware.json
  2. 40
      application/src/main/data/json/demo/dashboards/software.json
  3. 4
      application/src/main/data/json/system/widget_bundles/control_widgets.json
  4. 174
      application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql
  5. 14
      application/src/main/data/upgrade/3.2.2/schema_update.sql
  6. 90
      application/src/main/data/upgrade/3.2.2/schema_update_event.sql
  7. 23
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  8. 2
      application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java
  9. 159
      application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java
  10. 1
      application/src/main/java/org/thingsboard/server/actors/device/SessionInfo.java
  11. 4
      application/src/main/java/org/thingsboard/server/controller/AssetController.java
  12. 10
      application/src/main/java/org/thingsboard/server/controller/AuthController.java
  13. 48
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  14. 4
      application/src/main/java/org/thingsboard/server/controller/DashboardController.java
  15. 4
      application/src/main/java/org/thingsboard/server/controller/DeviceController.java
  16. 20
      application/src/main/java/org/thingsboard/server/controller/EdgeController.java
  17. 4
      application/src/main/java/org/thingsboard/server/controller/EntityViewController.java
  18. 9
      application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java
  19. 57
      application/src/main/java/org/thingsboard/server/controller/RpcController.java
  20. 21
      application/src/main/java/org/thingsboard/server/controller/RuleChainController.java
  21. 15
      application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java
  22. 2
      application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeNotificationService.java
  23. 34
      application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java
  24. 111
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java
  25. 300
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java
  26. 32
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSessionState.java
  27. 74
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java
  28. 10
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java
  29. 24
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AssetsEdgeEventFetcher.java
  30. 40
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/BasePageableEdgeEventFetcher.java
  31. 24
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/BaseUsersEdgeEventFetcher.java
  32. 24
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/BaseWidgetsBundlesEdgeEventFetcher.java
  33. 49
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/CustomerEdgeEventFetcher.java
  34. 3
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/CustomerUsersEdgeEventFetcher.java
  35. 24
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/DashboardsEdgeEventFetcher.java
  36. 24
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/DeviceProfilesEdgeEventFetcher.java
  37. 24
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/RuleChainsEdgeEventFetcher.java
  38. 4
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/TenantWidgetsBundlesEdgeEventFetcher.java
  39. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java
  40. 33
      application/src/main/java/org/thingsboard/server/service/executors/GrpcCallbackExecutorService.java
  41. 6
      application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
  42. 16
      application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java
  43. 27
      application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java
  44. 4
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  45. 82
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
  46. 2
      application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java
  47. 52
      application/src/main/java/org/thingsboard/server/service/lwm2m/LwM2MServerSecurityInfoRepository.java
  48. 72
      application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java
  49. 8
      application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java
  50. 256
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  51. 45
      application/src/main/java/org/thingsboard/server/service/queue/TbTopicWithConsumerPerPartition.java
  52. 11
      application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java
  53. 1
      application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java
  54. 2
      application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java
  55. 77
      application/src/main/java/org/thingsboard/server/service/rpc/TbRpcService.java
  56. 25
      application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java
  57. 6
      application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java
  58. 19
      application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java
  59. 2
      application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java
  60. 3
      application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java
  61. 1
      application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java
  62. 390
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
  63. 50
      application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java
  64. 44
      application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java
  65. 3
      application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java
  66. 33
      application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java
  67. 37
      application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java
  68. 29
      application/src/main/java/org/thingsboard/server/service/ttl/TimeseriesCleanUpService.java
  69. 83
      application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java
  70. 44
      application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java
  71. 36
      application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java
  72. 1
      application/src/main/resources/logback.xml
  73. 61
      application/src/main/resources/thingsboard.yml
  74. 58
      application/src/test/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessorTest.java
  75. 8
      application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
  76. 228
      application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java
  77. 37
      application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java
  78. 1
      application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java
  79. 1
      application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java
  80. 8
      application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java
  81. 54
      application/src/test/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaServiceTest.java
  82. 2
      application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java
  83. 83
      application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java
  84. 50
      application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java
  85. 34
      application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java
  86. 1
      application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java
  87. 8
      application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcProtoIntegrationTest.java
  88. 48
      application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesIntegrationTest.java
  89. 47
      application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesProtoIntegrationTest.java
  90. 44
      application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java
  91. 139
      application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java
  92. 146
      application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java
  93. 305
      application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java
  94. 43
      application/src/test/java/org/thingsboard/server/transport/lwm2m/PskLwm2mIntegrationTest.java
  95. 40
      application/src/test/java/org/thingsboard/server/transport/lwm2m/RpkLwM2MIntegrationTest.java
  96. 137
      application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java
  97. 155
      application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java
  98. 6
      application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java
  99. 54
      application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SimpleLwM2MDevice.java
  100. 155
      application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SwLwM2MDevice.java

40
application/src/main/data/json/demo/dashboards/firmware.json

@ -33,15 +33,15 @@
"enableSearch": true,
"displayPagination": true,
"defaultPageSize": 10,
"defaultSortOrder": "entityLabel",
"displayEntityName": false,
"defaultSortOrder": "entityName",
"displayEntityName": true,
"displayEntityType": false,
"enableSelectColumnDisplay": false,
"enableStickyHeader": true,
"enableStickyAction": false,
"entitiesTitle": "Devices",
"displayEntityLabel": true,
"entityLabelColumnTitle": "Device"
"displayEntityLabel": false,
"entityNameColumnTitle": "Device"
},
"title": "New Entities table",
"dropShadow": true,
@ -828,15 +828,15 @@
"enableSearch": true,
"displayPagination": true,
"defaultPageSize": 10,
"defaultSortOrder": "entityLabel",
"displayEntityName": false,
"defaultSortOrder": "entityName",
"displayEntityName": true,
"displayEntityType": false,
"enableSelectColumnDisplay": false,
"enableStickyHeader": true,
"enableStickyAction": true,
"entitiesTitle": "Devices",
"displayEntityLabel": true,
"entityLabelColumnTitle": "Device"
"displayEntityLabel": false,
"entityNameColumnTitle": "Device"
},
"title": "New Entities table",
"dropShadow": true,
@ -1125,15 +1125,15 @@
"enableSearch": true,
"displayPagination": true,
"defaultPageSize": 10,
"defaultSortOrder": "entityLabel",
"displayEntityName": false,
"defaultSortOrder": "entityName",
"displayEntityName": true,
"displayEntityType": false,
"enableSelectColumnDisplay": false,
"enableStickyHeader": true,
"enableStickyAction": true,
"entitiesTitle": "Devices",
"displayEntityLabel": true,
"entityLabelColumnTitle": "Device"
"displayEntityLabel": false,
"entityNameColumnTitle": "Device"
},
"title": "New Entities table",
"dropShadow": true,
@ -1422,15 +1422,15 @@
"enableSearch": true,
"displayPagination": true,
"defaultPageSize": 10,
"defaultSortOrder": "entityLabel",
"displayEntityName": false,
"defaultSortOrder": "entityName",
"displayEntityName": true,
"displayEntityType": false,
"enableSelectColumnDisplay": false,
"enableStickyHeader": true,
"enableStickyAction": true,
"entitiesTitle": "Devices",
"displayEntityLabel": true,
"entityLabelColumnTitle": "Device"
"displayEntityLabel": false,
"entityNameColumnTitle": "Device"
},
"title": "New Entities table",
"dropShadow": true,
@ -1719,15 +1719,15 @@
"enableSearch": true,
"displayPagination": true,
"defaultPageSize": 10,
"defaultSortOrder": "entityLabel",
"displayEntityName": false,
"defaultSortOrder": "entityName",
"displayEntityName": true,
"displayEntityType": false,
"enableSelectColumnDisplay": false,
"enableStickyHeader": true,
"enableStickyAction": true,
"entitiesTitle": "Devices",
"displayEntityLabel": true,
"entityLabelColumnTitle": "Device"
"displayEntityLabel": false,
"entityNameColumnTitle": "Device"
},
"title": "New Entities table",
"dropShadow": true,

40
application/src/main/data/json/demo/dashboards/software.json

@ -33,15 +33,15 @@
"enableSearch": true,
"displayPagination": true,
"defaultPageSize": 10,
"defaultSortOrder": "entityLabel",
"displayEntityName": false,
"defaultSortOrder": "entityName",
"displayEntityName": true,
"displayEntityType": false,
"enableSelectColumnDisplay": false,
"enableStickyHeader": true,
"enableStickyAction": false,
"entitiesTitle": "Devices",
"displayEntityLabel": true,
"entityLabelColumnTitle": "Device"
"displayEntityLabel": false,
"entityNameColumnTitle": "Device"
},
"title": "New Entities table",
"dropShadow": true,
@ -828,15 +828,15 @@
"enableSearch": true,
"displayPagination": true,
"defaultPageSize": 10,
"defaultSortOrder": "entityLabel",
"displayEntityName": false,
"defaultSortOrder": "entityName",
"displayEntityName": true,
"displayEntityType": false,
"enableSelectColumnDisplay": false,
"enableStickyHeader": true,
"enableStickyAction": true,
"entitiesTitle": "Devices",
"displayEntityLabel": true,
"entityLabelColumnTitle": "Device"
"displayEntityLabel": false,
"entityNameColumnTitle": "Device"
},
"title": "New Entities table",
"dropShadow": true,
@ -1125,15 +1125,15 @@
"enableSearch": true,
"displayPagination": true,
"defaultPageSize": 10,
"defaultSortOrder": "entityLabel",
"displayEntityName": false,
"defaultSortOrder": "entityName",
"displayEntityName": true,
"displayEntityType": false,
"enableSelectColumnDisplay": false,
"enableStickyHeader": true,
"enableStickyAction": true,
"entitiesTitle": "Devices",
"displayEntityLabel": true,
"entityLabelColumnTitle": "Device"
"displayEntityLabel": false,
"entityNameColumnTitle": "Device"
},
"title": "New Entities table",
"dropShadow": true,
@ -1422,15 +1422,15 @@
"enableSearch": true,
"displayPagination": true,
"defaultPageSize": 10,
"defaultSortOrder": "entityLabel",
"displayEntityName": false,
"defaultSortOrder": "entityName",
"displayEntityName": true,
"displayEntityType": false,
"enableSelectColumnDisplay": false,
"enableStickyHeader": true,
"enableStickyAction": true,
"entitiesTitle": "Devices",
"displayEntityLabel": true,
"entityLabelColumnTitle": "Device"
"displayEntityLabel": false,
"entityNameColumnTitle": "Device"
},
"title": "New Entities table",
"dropShadow": true,
@ -1719,15 +1719,15 @@
"enableSearch": true,
"displayPagination": true,
"defaultPageSize": 10,
"defaultSortOrder": "entityLabel",
"displayEntityName": false,
"defaultSortOrder": "entityName",
"displayEntityName": true,
"displayEntityType": false,
"enableSelectColumnDisplay": false,
"enableStickyHeader": true,
"enableStickyAction": true,
"entitiesTitle": "Devices",
"displayEntityLabel": true,
"entityLabelColumnTitle": "Device"
"displayEntityLabel": false,
"entityNameColumnTitle": "Device"
},
"title": "New Entities table",
"dropShadow": true,

4
application/src/main/data/json/system/widget_bundles/control_widgets.json

@ -18,8 +18,8 @@
"resources": [],
"templateHtml": "<div style=\"height: 100%; overflow-y: auto;\" id=\"device-terminal\"></div>",
"templateCss": ".cmd .cursor.blink {\n -webkit-animation-name: terminal-underline;\n -moz-animation-name: terminal-underline;\n -ms-animation-name: terminal-underline;\n animation-name: terminal-underline;\n}\n.terminal .inverted, .cmd .inverted {\n border-bottom-color: #aaa;\n}\n\n",
"controllerScript": "var requestTimeout = 500;\n\nself.onInit = function() {\n var subscription = self.ctx.defaultSubscription;\n var rpcEnabled = subscription.rpcEnabled;\n var deviceName = 'Simulated';\n var prompt;\n if (subscription.targetDeviceName && subscription.targetDeviceName.length) {\n deviceName = subscription.targetDeviceName;\n }\n if (self.ctx.settings.requestTimeout) {\n requestTimeout = self.ctx.settings.requestTimeout;\n }\n var greetings = 'Welcome to ThingsBoard RPC debug terminal.\\n\\n';\n if (!rpcEnabled) {\n greetings += 'Target device is not set!\\n\\n';\n prompt = '';\n } else {\n greetings += 'Current target device for RPC commands: [[b;#fff;]' + deviceName + ']\\n\\n';\n greetings += 'Please type [[b;#fff;]\\'help\\'] to see usage.\\n';\n prompt = '[[b;#8bc34a;]' + deviceName +']> ';\n }\n \n var terminal = $('#device-terminal', self.ctx.$container).terminal(\n function(command) {\n if (command !== '') {\n try {\n var localCommand = command.trim();\n var requestUUID = uuidv4();\n if (localCommand === 'help') {\n printUsage(this);\n } else {\n var spaceIndex = localCommand.indexOf(' ');\n if (spaceIndex === -1 && !localCommand.length) {\n this.error(\"Wrong number of arguments!\");\n this.echo(' ');\n } else {\n var params;\n if (spaceIndex === -1) {\n spaceIndex = localCommand.length;\n }\n var name = localCommand.substr(0, spaceIndex);\n var args = localCommand.substr(spaceIndex + 1);\n if (args.length) {\n try {\n params = JSON.parse(args);\n } catch (e) {\n params = args;\n }\n }\n performRpc(this, name, params, requestUUID);\n }\n }\n } catch(e) {\n this.error(new String(e));\n }\n } else {\n this.echo('');\n }\n }, {\n greetings: greetings,\n prompt: prompt,\n enabled: rpcEnabled\n });\n \n if (!rpcEnabled) {\n terminal.error('No RPC target detected!').pause();\n }\n}\n\n\nfunction printUsage(terminal) {\n var commandsListText = '\\n[[b;#fff;]Usage:]\\n';\n commandsListText += ' <method> [params body]]\\n\\n';\n commandsListText += '[[b;#fff;]Example 1:]\\n'; \n commandsListText += ' myRemoteMethod1 myText\\n\\n'; \n commandsListText += '[[b;#fff;]Example 2:]\\n'; \n commandsListText += ' myOtherRemoteMethod \"{\\\\\"key1\\\\\": 2, \\\\\"key2\\\\\": \\\\\"myVal\\\\\"}\"\\n'; \n terminal.echo(new String(commandsListText));\n}\n\n\nfunction performRpc(terminal, method, params, requestUUID) {\n terminal.pause();\n self.ctx.controlApi.sendTwoWayCommand(method, params, requestTimeout, requestUUID).subscribe(\n function success(responseBody) {\n terminal.echo(JSON.stringify(responseBody));\n terminal.echo(' ');\n terminal.resume();\n },\n function fail() {\n var errorText = self.ctx.defaultSubscription.rpcErrorText;\n terminal.error(errorText);\n terminal.echo(' ');\n terminal.resume();\n }\n );\n}\n\n\nfunction uuidv4() {\n return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {\n var r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8);\n return v.toString(16);\n });\n}\n\n \nself.onDestroy = function() {\n}",
"settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Settings\",\n \"properties\": {\n \"requestTimeout\": {\n \"title\": \"RPC request timeout (ms)\",\n \"type\": \"number\",\n \"default\": 500\n }\n },\n \"required\": [\"requestTimeout\"]\n },\n \"form\": [\n \"requestTimeout\"\n ]\n}",
"controllerScript": "var requestTimeout = 500;\nvar requestPersistent = false;\n\nself.onInit = function() {\n var subscription = self.ctx.defaultSubscription;\n var rpcEnabled = subscription.rpcEnabled;\n var deviceName = 'Simulated';\n var prompt;\n if (subscription.targetDeviceName && subscription.targetDeviceName.length) {\n deviceName = subscription.targetDeviceName;\n }\n if (self.ctx.settings.requestTimeout) {\n requestTimeout = self.ctx.settings.requestTimeout;\n }\n if (self.ctx.settings.requestPersistent) {\n requestPersistent = self.ctx.settings.requestPersistent;\n }\n var greetings = 'Welcome to ThingsBoard RPC debug terminal.\\n\\n';\n if (!rpcEnabled) {\n greetings += 'Target device is not set!\\n\\n';\n prompt = '';\n } else {\n greetings += 'Current target device for RPC commands: [[b;#fff;]' + deviceName + ']\\n\\n';\n greetings += 'Please type [[b;#fff;]\\'help\\'] to see usage.\\n';\n prompt = '[[b;#8bc34a;]' + deviceName +']> ';\n }\n \n var terminal = $('#device-terminal', self.ctx.$container).terminal(\n function(command) {\n if (command !== '') {\n try {\n var localCommand = command.trim();\n var requestUUID = uuidv4();\n if (localCommand === 'help') {\n printUsage(this);\n } else {\n var spaceIndex = localCommand.indexOf(' ');\n if (spaceIndex === -1 && !localCommand.length) {\n this.error(\"Wrong number of arguments!\");\n this.echo(' ');\n } else {\n var params;\n if (spaceIndex === -1) {\n spaceIndex = localCommand.length;\n }\n var name = localCommand.substr(0, spaceIndex);\n var args = localCommand.substr(spaceIndex + 1);\n if (args.length) {\n try {\n params = JSON.parse(args);\n } catch (e) {\n params = args;\n }\n }\n performRpc(this, name, params, requestUUID);\n }\n }\n } catch(e) {\n this.error(new String(e));\n }\n } else {\n this.echo('');\n }\n }, {\n greetings: greetings,\n prompt: prompt,\n enabled: rpcEnabled\n });\n \n if (!rpcEnabled) {\n terminal.error('No RPC target detected!').pause();\n }\n}\n\n\nfunction printUsage(terminal) {\n var commandsListText = '\\n[[b;#fff;]Usage:]\\n';\n commandsListText += ' <method> [params body]]\\n\\n';\n commandsListText += '[[b;#fff;]Example 1:]\\n'; \n commandsListText += ' myRemoteMethod1 myText\\n\\n'; \n commandsListText += '[[b;#fff;]Example 2:]\\n'; \n commandsListText += ' myOtherRemoteMethod \"{\\\\\"key1\\\\\": 2, \\\\\"key2\\\\\": \\\\\"myVal\\\\\"}\"\\n'; \n terminal.echo(new String(commandsListText));\n}\n\n\nfunction performRpc(terminal, method, params, requestUUID) {\n terminal.pause();\n self.ctx.controlApi.sendTwoWayCommand(method, params, requestTimeout, requestPersistent, requestUUID).subscribe(\n function success(responseBody) {\n terminal.echo(JSON.stringify(responseBody));\n terminal.echo(' ');\n terminal.resume();\n },\n function fail() {\n var errorText = self.ctx.defaultSubscription.rpcErrorText;\n terminal.error(errorText);\n terminal.echo(' ');\n terminal.resume();\n }\n );\n}\n\n\nfunction uuidv4() {\n return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {\n var r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8);\n return v.toString(16);\n });\n}\n\n \nself.onDestroy = function() {\n}",
"settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Settings\",\n \"properties\": {\n \"requestTimeout\": {\n \"title\": \"RPC request timeout (ms)\",\n \"type\": \"number\",\n \"default\": 500\n },\n \"requestPersistent\": {\n \"title\": \"RPC request persistent\",\n \"type\": \"boolean\",\n \"default\": false\n }\n },\n \"required\": [\"requestTimeout\"]\n },\n \"form\": [\n \"requestTimeout\",\n \"requestPersistent\"\n ]\n}",
"dataKeySettingsSchema": "{}\n",
"defaultConfig": "{\"targetDeviceAliases\":[],\"showTitle\":true,\"backgroundColor\":\"#010101\",\"color\":\"rgba(255, 254, 254, 0.87)\",\"padding\":\"0px\",\"settings\":{\"parseGpioStatusFunction\":\"return body[pin] === true;\",\"gpioStatusChangeRequest\":{\"method\":\"setGpioStatus\",\"paramsBody\":\"{\\n \\\"pin\\\": \\\"{$pin}\\\",\\n \\\"enabled\\\": \\\"{$enabled}\\\"\\n}\"},\"requestTimeout\":500,\"switchPanelBackgroundColor\":\"#b71c1c\",\"gpioStatusRequest\":{\"method\":\"getGpioStatus\",\"paramsBody\":\"{}\"},\"gpioList\":[{\"pin\":1,\"label\":\"GPIO 1\",\"row\":0,\"col\":0,\"_uniqueKey\":0},{\"pin\":2,\"label\":\"GPIO 2\",\"row\":0,\"col\":1,\"_uniqueKey\":1},{\"pin\":3,\"label\":\"GPIO 3\",\"row\":1,\"col\":0,\"_uniqueKey\":2}]},\"title\":\"RPC debug terminal\",\"dropShadow\":true,\"enableFullscreen\":true,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"showLegend\":false,\"actions\":{}}"
}

174
application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql

@ -18,17 +18,18 @@ CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar
LANGUAGE plpgsql AS
$$
DECLARE
max_tenant_ttl bigint;
max_customer_ttl bigint;
max_ttl bigint;
date timestamp;
partition_by_max_ttl_date varchar;
partition_month varchar;
partition_day varchar;
partition_year varchar;
partition varchar;
partition_to_delete varchar;
max_tenant_ttl bigint;
max_customer_ttl bigint;
max_ttl bigint;
date timestamp;
partition_by_max_ttl_date varchar;
partition_by_max_ttl_month varchar;
partition_by_max_ttl_day varchar;
partition_by_max_ttl_year varchar;
partition varchar;
partition_year integer;
partition_month integer;
partition_day integer;
BEGIN
SELECT max(attribute_kv.long_v)
@ -45,53 +46,138 @@ BEGIN
if max_ttl IS NOT NULL AND max_ttl > 0 THEN
date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - max_ttl);
partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date);
RAISE NOTICE 'Date by max ttl: %', date;
RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date;
IF partition_by_max_ttl_date IS NOT NULL THEN
CASE
WHEN partition_type = 'DAYS' THEN
partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
partition_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5);
partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
partition_by_max_ttl_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
partition_by_max_ttl_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5);
WHEN partition_type = 'MONTHS' THEN
partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
partition_by_max_ttl_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
ELSE
partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
END CASE;
FOR partition IN SELECT tablename
FROM pg_tables
WHERE schemaname = 'public'
AND tablename like 'ts_kv_' || '%'
AND tablename != 'ts_kv_latest'
AND tablename != 'ts_kv_dictionary'
AND tablename != 'ts_kv_indefinite'
LOOP
IF partition != partition_by_max_ttl_date THEN
IF partition_year IS NOT NULL THEN
IF SPLIT_PART(partition, '_', 3)::integer < partition_year::integer THEN
partition_to_delete := partition;
ELSE
IF partition_month IS NOT NULL THEN
IF SPLIT_PART(partition, '_', 4)::integer < partition_month::integer THEN
partition_to_delete := partition;
IF partition_by_max_ttl_year IS NULL THEN
RAISE NOTICE 'Failed to remove partitions by max ttl date due to partition_by_max_ttl_year is null!';
ELSE
IF partition_type = 'YEARS' THEN
FOR partition IN SELECT tablename
FROM pg_tables
WHERE schemaname = 'public'
AND tablename like 'ts_kv_' || '%'
AND tablename != 'ts_kv_latest'
AND tablename != 'ts_kv_dictionary'
AND tablename != 'ts_kv_indefinite'
AND tablename != partition_by_max_ttl_date
LOOP
partition_year := SPLIT_PART(partition, '_', 3)::integer;
IF partition_year < partition_by_max_ttl_year::integer THEN
RAISE NOTICE 'Partition to delete by max ttl: %', partition;
EXECUTE format('DROP TABLE IF EXISTS %I', partition);
deleted := deleted + 1;
END IF;
END LOOP;
ELSE
IF partition_type = 'MONTHS' THEN
IF partition_by_max_ttl_month IS NULL THEN
RAISE NOTICE 'Failed to remove months partitions by max ttl date due to partition_by_max_ttl_month is null!';
ELSE
FOR partition IN SELECT tablename
FROM pg_tables
WHERE schemaname = 'public'
AND tablename like 'ts_kv_' || '%'
AND tablename != 'ts_kv_latest'
AND tablename != 'ts_kv_dictionary'
AND tablename != 'ts_kv_indefinite'
AND tablename != partition_by_max_ttl_date
LOOP
partition_year := SPLIT_PART(partition, '_', 3)::integer;
IF partition_year > partition_by_max_ttl_year::integer THEN
RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition;
CONTINUE;
ELSE
IF partition_day IS NOT NULL THEN
IF SPLIT_PART(partition, '_', 5)::integer < partition_day::integer THEN
partition_to_delete := partition;
IF partition_year < partition_by_max_ttl_year::integer THEN
RAISE NOTICE 'Partition to delete by max ttl: %', partition;
EXECUTE format('DROP TABLE IF EXISTS %I', partition);
deleted := deleted + 1;
ELSE
partition_month := SPLIT_PART(partition, '_', 4)::integer;
IF partition_year = partition_by_max_ttl_year::integer THEN
IF partition_month >= partition_by_max_ttl_month::integer THEN
RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition;
CONTINUE;
ELSE
RAISE NOTICE 'Partition to delete by max ttl: %', partition;
EXECUTE format('DROP TABLE IF EXISTS %I', partition);
deleted := deleted + 1;
END IF;
END IF;
END IF;
END IF;
END LOOP;
END IF;
ELSE
IF partition_type = 'DAYS' THEN
IF partition_by_max_ttl_month IS NULL THEN
RAISE NOTICE 'Failed to remove days partitions by max ttl date due to partition_by_max_ttl_month is null!';
ELSE
IF partition_by_max_ttl_day IS NULL THEN
RAISE NOTICE 'Failed to remove days partitions by max ttl date due to partition_by_max_ttl_day is null!';
ELSE
FOR partition IN SELECT tablename
FROM pg_tables
WHERE schemaname = 'public'
AND tablename like 'ts_kv_' || '%'
AND tablename != 'ts_kv_latest'
AND tablename != 'ts_kv_dictionary'
AND tablename != 'ts_kv_indefinite'
AND tablename != partition_by_max_ttl_date
LOOP
partition_year := SPLIT_PART(partition, '_', 3)::integer;
IF partition_year > partition_by_max_ttl_year::integer THEN
RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition;
CONTINUE;
ELSE
IF partition_year < partition_by_max_ttl_year::integer THEN
RAISE NOTICE 'Partition to delete by max ttl: %', partition;
EXECUTE format('DROP TABLE IF EXISTS %I', partition);
deleted := deleted + 1;
ELSE
partition_month := SPLIT_PART(partition, '_', 4)::integer;
IF partition_month > partition_by_max_ttl_month::integer THEN
RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition;
CONTINUE;
ELSE
IF partition_month < partition_by_max_ttl_month::integer THEN
RAISE NOTICE 'Partition to delete by max ttl: %', partition;
EXECUTE format('DROP TABLE IF EXISTS %I', partition);
deleted := deleted + 1;
ELSE
partition_day := SPLIT_PART(partition, '_', 5)::integer;
IF partition_day >= partition_by_max_ttl_day::integer THEN
RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition;
CONTINUE;
ELSE
IF partition_day < partition_by_max_ttl_day::integer THEN
RAISE NOTICE 'Partition to delete by max ttl: %', partition;
EXECUTE format('DROP TABLE IF EXISTS %I', partition);
deleted := deleted + 1;
END IF;
END IF;
END IF;
END IF;
END IF;
END IF;
END LOOP;
END IF;
END IF;
END IF;
IF partition_to_delete IS NOT NULL THEN
RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete;
EXECUTE format('DROP TABLE IF EXISTS %I', partition_to_delete);
partition_to_delete := NULL;
deleted := deleted + 1;
END IF;
END IF;
END LOOP;
END IF;
END IF;
END IF;
END IF;
END
@ -107,8 +193,6 @@ BEGIN
partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM');
WHEN partition_type = 'YEARS' THEN
partition := 'ts_kv_' || to_char(date, 'yyyy');
WHEN partition_type = 'INDEFINITE' THEN
partition := NULL;
ELSE
partition := NULL;
END CASE;

14
application/src/main/data/upgrade/3.2.2/schema_update.sql

@ -197,3 +197,17 @@ $$;
ALTER TABLE api_usage_state
ADD COLUMN IF NOT EXISTS alarm_exec VARCHAR(32);
UPDATE api_usage_state SET alarm_exec = 'ENABLED' WHERE alarm_exec IS NULL;
CREATE TABLE IF NOT EXISTS rpc (
id uuid NOT NULL CONSTRAINT rpc_pkey PRIMARY KEY,
created_time bigint NOT NULL,
tenant_id uuid NOT NULL,
device_id uuid NOT NULL,
expiration_time bigint NOT NULL,
request varchar(10000000) NOT NULL,
response varchar(10000000),
status varchar(255) NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id);

90
application/src/main/data/upgrade/3.2.2/schema_update_event.sql

@ -0,0 +1,90 @@
--
-- Copyright © 2016-2021 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-- PROCEDURE: public.cleanup_events_by_ttl(bigint, bigint, bigint)
DROP PROCEDURE IF EXISTS public.cleanup_events_by_ttl(bigint, bigint, bigint);
CREATE OR REPLACE PROCEDURE public.cleanup_events_by_ttl(
ttl bigint,
debug_ttl bigint,
INOUT deleted bigint)
LANGUAGE 'plpgsql'
AS $BODY$
DECLARE
ttl_ts bigint;
debug_ttl_ts bigint;
ttl_deleted_count bigint DEFAULT 0;
debug_ttl_deleted_count bigint DEFAULT 0;
BEGIN
IF ttl > 0 THEN
ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint;
DELETE FROM event
WHERE ts < ttl_ts
AND NOT event_type IN ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN', 'DEBUG_CONVERTER', 'DEBUG_INTEGRATION');
GET DIAGNOSTICS ttl_deleted_count = ROW_COUNT;
END IF;
IF debug_ttl > 0 THEN
debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint;
DELETE FROM event
WHERE ts < debug_ttl_ts
AND event_type IN ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN', 'DEBUG_CONVERTER', 'DEBUG_INTEGRATION');
GET DIAGNOSTICS debug_ttl_deleted_count = ROW_COUNT;
END IF;
RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count;
RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count;
deleted := ttl_deleted_count + debug_ttl_deleted_count;
END
$BODY$;
-- Index: idx_event_ts
DROP INDEX IF EXISTS public.idx_event_ts;
-- Hint: add CONCURRENTLY to CREATE INDEX query in case of more then 1 million records or during live update
-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_ts
CREATE INDEX IF NOT EXISTS idx_event_ts
ON public.event
(ts DESC NULLS LAST)
WITH (FILLFACTOR=95);
COMMENT ON INDEX public.idx_event_ts
IS 'This index helps to delete events by TTL using timestamp';
-- Index: idx_event_tenant_entity_type_entity_event_type_created_time_des
DROP INDEX IF EXISTS public.idx_event_tenant_entity_type_entity_event_type_created_time_des;
-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des
CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des
ON public.event
(tenant_id ASC, entity_type ASC, entity_id ASC, event_type ASC, created_time DESC NULLS LAST)
WITH (FILLFACTOR=95);
COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des
IS 'This index helps to open latest events on UI fast';
-- Index: idx_event_type_entity_id
-- Description: replaced with more suitable idx_event_tenant_entity_type_entity_event_type_created_time_des
DROP INDEX IF EXISTS public.idx_event_type_entity_id;

23
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -65,6 +65,7 @@ import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.rule.RuleNodeStateService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
@ -80,9 +81,9 @@ import org.thingsboard.server.service.executors.ExternalCallExecutorService;
import org.thingsboard.server.service.executors.SharedEventLoopGroupService;
import org.thingsboard.server.service.mail.MailExecutorService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
import org.thingsboard.server.service.rpc.TbRpcService;
import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
import org.thingsboard.server.service.script.JsExecutorService;
import org.thingsboard.server.service.script.JsInvokeService;
@ -303,23 +304,33 @@ public class ActorSystemContext {
@Lazy
@Autowired(required = false)
@Getter private EdgeService edgeService;
@Getter
private EdgeService edgeService;
@Lazy
@Autowired(required = false)
@Getter private EdgeEventService edgeEventService;
@Getter
private EdgeEventService edgeEventService;
@Lazy
@Autowired(required = false)
@Getter private EdgeRpcService edgeRpcService;
@Getter
private EdgeRpcService edgeRpcService;
@Lazy
@Autowired(required = false)
@Getter private ResourceService resourceService;
@Getter
private ResourceService resourceService;
@Lazy
@Autowired(required = false)
@Getter private OtaPackageService otaPackageService;
@Getter
private OtaPackageService otaPackageService;
@Lazy
@Autowired(required = false)
@Getter
private TbRpcService tbRpcService;
@Value("${actors.session.max_concurrent_sessions_per_device:1}")
@Getter

2
application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java

@ -46,7 +46,7 @@ public class DeviceActor extends ContextAwareActor {
super.init(ctx);
log.debug("[{}][{}] Starting device actor.", processor.tenantId, processor.deviceId);
try {
processor.initSessionTimeout(ctx);
processor.init(ctx);
log.debug("[{}][{}] Device actor started.", processor.tenantId, processor.deviceId);
} catch (Exception e) {
log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.deviceId, e);

159
application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java

@ -23,6 +23,8 @@ import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.LinkedHashMapRemoveEldest;
import org.thingsboard.rule.engine.api.RpcError;
import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg;
import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg;
@ -38,12 +40,17 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.RpcId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKey;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.rpc.Rpc;
import org.thingsboard.server.common.data.rpc.RpcStatus;
import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
@ -52,8 +59,8 @@ import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest;
import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry;
import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg;
@ -68,10 +75,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.SessionType;
import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg;
import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg;
import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToDevicePersistedRpcResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportUpdateCredentialsProto;
import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto;
import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
@ -88,6 +97,8 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.function.Consumer;
@ -102,7 +113,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
final TenantId tenantId;
final DeviceId deviceId;
private final Map<UUID, SessionInfoMetaData> sessions;
final LinkedHashMapRemoveEldest<UUID, SessionInfoMetaData> sessions;
private final Map<UUID, SessionInfo> attributeSubscriptions;
private final Map<UUID, SessionInfo> rpcSubscriptions;
private final Map<Integer, ToDeviceRpcRequestMetadata> toDeviceRpcPendingMap;
@ -117,16 +128,16 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
super(systemContext);
this.tenantId = tenantId;
this.deviceId = deviceId;
this.sessions = new LinkedHashMap<>();
this.attributeSubscriptions = new HashMap<>();
this.rpcSubscriptions = new HashMap<>();
this.toDeviceRpcPendingMap = new HashMap<>();
this.sessions = new LinkedHashMapRemoveEldest<>(systemContext.getMaxConcurrentSessionsPerDevice(), this::notifyTransportAboutClosedSessionMaxSessionsLimit);
if (initAttributes()) {
restoreSessions();
}
}
private boolean initAttributes() {
boolean initAttributes() {
Device device = systemContext.getDeviceService().findDeviceById(tenantId, deviceId);
if (device != null) {
this.deviceName = device.getName();
@ -162,20 +173,19 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
void processRpcRequest(TbActorCtx context, ToDeviceRpcRequestActorMsg msg) {
ToDeviceRpcRequest request = msg.getMsg();
ToDeviceRpcRequestBody body = request.getBody();
ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder()
.setRequestId(rpcSeq++)
.setMethodName(body.getMethod())
.setParams(body.getParams())
.setExpirationTime(request.getExpirationTime())
.setRequestIdMSB(request.getId().getMostSignificantBits())
.setRequestIdLSB(request.getId().getLeastSignificantBits())
.build();
ToDeviceRpcRequestMsg rpcRequest = creteToDeviceRpcRequestMsg(request);
long timeout = request.getExpirationTime() - System.currentTimeMillis();
boolean persisted = request.isPersisted();
if (timeout <= 0) {
log.debug("[{}][{}] Ignoring message due to exp time reached, {}", deviceId, request.getId(), request.getExpirationTime());
if (persisted) {
createRpc(request, RpcStatus.TIMEOUT);
}
return;
} else if (persisted) {
createRpc(request, RpcStatus.QUEUED);
}
boolean sent;
@ -192,11 +202,17 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
syncSessionSet.add(key);
}
});
log.trace("46) Rpc syncSessionSet [{}] subscription after sent [{}]",syncSessionSet, rpcSubscriptions);
log.trace("46) Rpc syncSessionSet [{}] subscription after sent [{}]", syncSessionSet, rpcSubscriptions);
syncSessionSet.forEach(rpcSubscriptions::remove);
}
if (request.isOneway() && sent) {
if (persisted) {
ObjectNode response = JacksonUtil.newObjectNode();
response.put("rpcId", request.getId().toString());
systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), JacksonUtil.toString(response), null));
}
if (!persisted && request.isOneway() && sent) {
log.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId());
systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null));
} else {
@ -209,6 +225,31 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
}
}
private Rpc createRpc(ToDeviceRpcRequest request, RpcStatus status) {
Rpc rpc = new Rpc(new RpcId(request.getId()));
rpc.setCreatedTime(System.currentTimeMillis());
rpc.setTenantId(tenantId);
rpc.setDeviceId(deviceId);
rpc.setExpirationTime(request.getExpirationTime());
rpc.setRequest(JacksonUtil.valueToTree(request));
rpc.setStatus(status);
return systemContext.getTbRpcService().save(tenantId, rpc);
}
private ToDeviceRpcRequestMsg creteToDeviceRpcRequestMsg(ToDeviceRpcRequest request) {
ToDeviceRpcRequestBody body = request.getBody();
return ToDeviceRpcRequestMsg.newBuilder()
.setRequestId(rpcSeq++)
.setMethodName(body.getMethod())
.setParams(body.getParams())
.setExpirationTime(request.getExpirationTime())
.setRequestIdMSB(request.getId().getMostSignificantBits())
.setRequestIdLSB(request.getId().getLeastSignificantBits())
.setOneway(request.isOneway())
.setPersisted(request.isPersisted())
.build();
}
void processRpcResponsesFromEdge(TbActorCtx context, FromDeviceRpcResponseActorMsg responseMsg) {
log.debug("[{}] Processing rpc command response from edge session", deviceId);
ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(responseMsg.getRequestId());
@ -230,6 +271,9 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(msg.getId());
if (requestMd != null) {
log.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId());
if (requestMd.getMsg().getMsg().isPersisted()) {
systemContext.getTbRpcService().save(tenantId, new RpcId(requestMd.getMsg().getMsg().getId()), RpcStatus.TIMEOUT, null);
}
systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(),
null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION));
}
@ -253,7 +297,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
toDeviceRpcPendingMap.entrySet().stream().findFirst().ifPresent(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds));
}
sentOneWayIds.forEach(toDeviceRpcPendingMap::remove);
sentOneWayIds.stream().filter(id -> !toDeviceRpcPendingMap.get(id).getMsg().getMsg().isPersisted()).forEach(toDeviceRpcPendingMap::remove);
}
private Consumer<Map.Entry<Integer, ToDeviceRpcRequestMetadata>> processPendingRpc(TbActorCtx context, UUID sessionId, String nodeId, Set<Integer> sentOneWayIds) {
@ -271,7 +315,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
.setExpirationTime(request.getExpirationTime())
.setRequestIdMSB(request.getId().getMostSignificantBits())
.setRequestIdLSB(request.getId().getLeastSignificantBits())
.setOneway(request.isOneway())
.setPersisted(request.isPersisted())
.build();
sendToTransport(rpcRequest, sessionId, nodeId);
};
}
@ -279,31 +326,39 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
void process(TbActorCtx context, TransportToDeviceActorMsgWrapper wrapper) {
TransportToDeviceActorMsg msg = wrapper.getMsg();
TbCallback callback = wrapper.getCallback();
var sessionInfo = msg.getSessionInfo();
if (msg.hasSessionEvent()) {
processSessionStateMsgs(msg.getSessionInfo(), msg.getSessionEvent());
processSessionStateMsgs(sessionInfo, msg.getSessionEvent());
}
if (msg.hasSubscribeToAttributes()) {
processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToAttributes());
processSubscriptionCommands(context, sessionInfo, msg.getSubscribeToAttributes());
}
if (msg.hasSubscribeToRPC()) {
processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToRPC());
processSubscriptionCommands(context, sessionInfo, msg.getSubscribeToRPC());
}
if (msg.hasSendPendingRPC()) {
sendPendingRequests(context, getSessionId(sessionInfo), sessionInfo);
}
if (msg.hasGetAttributes()) {
handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes());
handleGetAttributesRequest(context, sessionInfo, msg.getGetAttributes());
}
if (msg.hasToDeviceRPCCallResponse()) {
processRpcResponses(context, msg.getSessionInfo(), msg.getToDeviceRPCCallResponse());
processRpcResponses(context, sessionInfo, msg.getToDeviceRPCCallResponse());
}
if (msg.hasSubscriptionInfo()) {
handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo());
handleSessionActivity(context, sessionInfo, msg.getSubscriptionInfo());
}
if (msg.hasClaimDevice()) {
handleClaimDeviceMsg(context, msg.getSessionInfo(), msg.getClaimDevice());
handleClaimDeviceMsg(context, sessionInfo, msg.getClaimDevice());
}
if (msg.hasPersistedRpcResponseMsg()) {
processPersistedRpcResponses(context, sessionInfo, msg.getPersistedRpcResponseMsg());
}
callback.onSuccess();
}
private void handleClaimDeviceMsg(TbActorCtx context, SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg) {
private void handleClaimDeviceMsg(TbActorCtx context, SessionInfoProto sessionInfo, ClaimDeviceMsg msg) {
DeviceId deviceId = new DeviceId(new UUID(msg.getDeviceIdMSB(), msg.getDeviceIdLSB()));
systemContext.getClaimDevicesService().registerClaimingInfo(tenantId, deviceId, msg.getSecretKey(), msg.getDurationMs());
}
@ -442,11 +497,19 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
if (success) {
systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(),
responseMsg.getPayload(), null));
if (requestMd.getMsg().getMsg().isPersisted()) {
systemContext.getTbRpcService().save(tenantId, new RpcId(requestMd.getMsg().getMsg().getId()), RpcStatus.SUCCESSFUL, JacksonUtil.toJsonNode(responseMsg.getPayload()));
}
} else {
log.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId());
}
}
private void processPersistedRpcResponses(TbActorCtx context, SessionInfoProto sessionInfo, ToDevicePersistedRpcResponseMsg responseMsg) {
UUID rpcId = new UUID(responseMsg.getRequestIdMSB(), responseMsg.getRequestIdLSB());
systemContext.getTbRpcService().save(tenantId, new RpcId(rpcId), RpcStatus.valueOf(responseMsg.getStatus()), null);
}
private void processSubscriptionCommands(TbActorCtx context, SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg subscribeCmd) {
UUID sessionId = getSessionId(sessionInfo);
if (subscribeCmd.getUnsubscribe()) {
@ -488,18 +551,14 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
private void processSessionStateMsgs(SessionInfoProto sessionInfo, SessionEventMsg msg) {
UUID sessionId = getSessionId(sessionInfo);
Objects.requireNonNull(sessionId);
if (msg.getEvent() == SessionEvent.OPEN) {
if (sessions.containsKey(sessionId)) {
log.debug("[{}] Received duplicate session open event [{}]", deviceId, sessionId);
return;
}
log.debug("[{}] Processing new session [{}]", deviceId, sessionId);
if (sessions.size() >= systemContext.getMaxConcurrentSessionsPerDevice()) {
UUID sessionIdToRemove = sessions.keySet().stream().findFirst().orElse(null);
if (sessionIdToRemove != null) {
notifyTransportAboutClosedSession(sessionIdToRemove, sessions.remove(sessionIdToRemove), "max concurrent sessions limit reached per device!");
}
}
log.info("[{}] Processing new session [{}]. Current sessions size {}", deviceId, sessionId, sessions.size());
sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfo.getNodeId())));
if (sessions.size() == 1) {
reportSessionOpen();
@ -520,8 +579,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
private void handleSessionActivity(TbActorCtx context, SessionInfoProto sessionInfoProto, SubscriptionInfoProto subscriptionInfo) {
UUID sessionId = getSessionId(sessionInfoProto);
Objects.requireNonNull(sessionId);
SessionInfoMetaData sessionMD = sessions.computeIfAbsent(sessionId,
id -> new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L));
id -> new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()), subscriptionInfo.getLastActivityTime()));
sessionMD.setLastActivityTime(subscriptionInfo.getLastActivityTime());
sessionMD.setSubscribedToAttributes(subscriptionInfo.getAttributeSubscription());
@ -551,6 +612,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
}
}
private void notifyTransportAboutClosedSessionMaxSessionsLimit(UUID sessionId, SessionInfoMetaData sessionMd) {
notifyTransportAboutClosedSession(sessionId, sessionMd, "max concurrent sessions limit reached per device!");
}
private void notifyTransportAboutClosedSession(UUID sessionId, SessionInfoMetaData sessionMd, String message) {
SessionCloseNotificationProto sessionCloseNotificationProto = SessionCloseNotificationProto
.newBuilder()
@ -565,7 +631,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
void notifyTransportAboutProfileUpdate(UUID sessionId, SessionInfoMetaData sessionMd, DeviceCredentials deviceCredentials) {
log.info("2) LwM2Mtype: ");
TransportProtos.ToTransportUpdateCredentialsProto.Builder notification = TransportProtos.ToTransportUpdateCredentialsProto.newBuilder();
ToTransportUpdateCredentialsProto.Builder notification = ToTransportUpdateCredentialsProto.newBuilder();
notification.addCredentialsId(deviceCredentials.getCredentialsId());
notification.addCredentialsValue(deviceCredentials.getCredentialsValue());
ToTransportMsg msg = ToTransportMsg.newBuilder()
@ -640,7 +706,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
ListenableFuture<EdgeEvent> future = systemContext.getEdgeEventService().saveAsync(edgeEvent);
Futures.addCallback(future, new FutureCallback<EdgeEvent>() {
@Override
public void onSuccess( EdgeEvent result) {
public void onSuccess(EdgeEvent result) {
systemContext.getClusterService().onEdgeEventUpdate(tenantId, edgeId);
}
@ -697,7 +763,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
return builder.build();
}
private void restoreSessions() {
void restoreSessions() {
log.debug("[{}] Restoring sessions from cache", deviceId);
DeviceSessionsCacheEntry sessionsDump = null;
try {
@ -710,6 +776,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
log.debug("[{}] No session information found", deviceId);
return;
}
// TODO: Take latest max allowed sessions size from cache
for (SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) {
SessionInfoProto sessionInfoProto = sessionSubscriptionInfoProto.getSessionInfo();
UUID sessionId = getSessionId(sessionInfoProto);
@ -756,11 +823,30 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
.addAllSessions(sessionsList).build().toByteArray());
}
void initSessionTimeout(TbActorCtx ctx) {
void init(TbActorCtx ctx) {
schedulePeriodicMsgWithDelay(ctx, SessionTimeoutCheckMsg.instance(), systemContext.getSessionReportTimeout(), systemContext.getSessionReportTimeout());
PageLink pageLink = new PageLink(1024);
PageData<Rpc> pageData;
do {
pageData = systemContext.getTbRpcService().findAllByDeviceIdAndStatus(tenantId, deviceId, RpcStatus.QUEUED, pageLink);
pageData.getData().forEach(rpc -> {
ToDeviceRpcRequest msg = JacksonUtil.convertValue(rpc.getRequest(), ToDeviceRpcRequest.class);
long timeout = rpc.getExpirationTime() - System.currentTimeMillis();
if (timeout <= 0) {
rpc.setStatus(RpcStatus.TIMEOUT);
systemContext.getTbRpcService().save(tenantId, rpc);
} else {
registerPendingRpcRequest(ctx, new ToDeviceRpcRequestActorMsg(systemContext.getServiceId(), msg), false, creteToDeviceRpcRequestMsg(msg), timeout);
}
});
if (pageData.hasNext()) {
pageLink = pageLink.nextPageLink();
}
} while (pageData.hasNext());
}
void checkSessionsTimeout() {
log.debug("[{}] checkSessionsTimeout started. Size before check {}", deviceId, sessions.size());
long expTime = System.currentTimeMillis() - systemContext.getSessionInactivityTimeout();
Map<UUID, SessionInfoMetaData> sessionsToRemove = sessions.entrySet().stream().filter(kv -> kv.getValue().getLastActivityTime() < expTime).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
sessionsToRemove.forEach((sessionId, sessionMD) -> {
@ -772,6 +858,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
if (!sessionsToRemove.isEmpty()) {
dumpSessions();
}
log.debug("[{}] checkSessionsTimeout finished. Size after check {}", deviceId, sessions.size());
}
}

1
application/src/main/java/org/thingsboard/server/actors/device/SessionInfo.java

@ -25,5 +25,4 @@ import org.thingsboard.server.gen.transport.TransportProtos.SessionType;
public class SessionInfo {
private final SessionType type;
private final String nodeId;
private long lastActivityTime;
}

4
application/src/main/java/org/thingsboard/server/controller/AssetController.java

@ -434,7 +434,7 @@ public class AssetController extends BaseController {
Edge edge = checkEdgeId(edgeId, Operation.READ);
AssetId assetId = new AssetId(toUUID(strAssetId));
checkAssetId(assetId, Operation.ASSIGN_TO_EDGE);
checkAssetId(assetId, Operation.READ);
Asset savedAsset = checkNotNull(assetService.assignAssetToEdge(getTenantId(), assetId, edgeId));
@ -467,7 +467,7 @@ public class AssetController extends BaseController {
Edge edge = checkEdgeId(edgeId, Operation.READ);
AssetId assetId = new AssetId(toUUID(strAssetId));
Asset asset = checkAssetId(assetId, Operation.UNASSIGN_FROM_EDGE);
Asset asset = checkAssetId(assetId, Operation.READ);
Asset savedAsset = checkNotNull(assetService.unassignAssetFromEdge(getTenantId(), assetId, edgeId));

10
application/src/main/java/org/thingsboard/server/controller/AuthController.java

@ -135,7 +135,7 @@ public class AuthController extends BaseController {
}
}
@RequestMapping(value = "/noauth/activate", params = { "activateToken" }, method = RequestMethod.GET)
@RequestMapping(value = "/noauth/activate", params = {"activateToken"}, method = RequestMethod.GET)
public ResponseEntity<String> checkActivateToken(
@RequestParam(value = "activateToken") String activateToken) {
HttpHeaders headers = new HttpHeaders();
@ -159,7 +159,7 @@ public class AuthController extends BaseController {
@RequestMapping(value = "/noauth/resetPasswordByEmail", method = RequestMethod.POST)
@ResponseStatus(value = HttpStatus.OK)
public void requestResetPasswordByEmail (
public void requestResetPasswordByEmail(
@RequestBody JsonNode resetPasswordByEmailRequest,
HttpServletRequest request) throws ThingsboardException {
try {
@ -170,13 +170,13 @@ public class AuthController extends BaseController {
String resetUrl = String.format("%s/api/noauth/resetPassword?resetToken=%s", baseUrl,
userCredentials.getResetToken());
mailService.sendResetPasswordEmail(resetUrl, email);
mailService.sendResetPasswordEmailAsync(resetUrl, email);
} catch (Exception e) {
throw handleException(e);
log.warn("Error occurred: {}", e.getMessage());
}
}
@RequestMapping(value = "/noauth/resetPassword", params = { "resetToken" }, method = RequestMethod.GET)
@RequestMapping(value = "/noauth/resetPassword", params = {"resetToken"}, method = RequestMethod.GET)
public ResponseEntity<String> checkResetToken(
@RequestParam(value = "resetToken") String resetToken) {
HttpHeaders headers = new HttpHeaders();

48
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -37,10 +37,10 @@ import org.thingsboard.server.common.data.EdgeUtils;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.EntityViewInfo;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.common.data.Tenant;
@ -69,7 +69,7 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.RpcId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TbResourceId;
@ -78,13 +78,14 @@ import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.id.WidgetTypeId;
import org.thingsboard.server.common.data.id.WidgetsBundleId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageDataIterableByTenantIdEntityId;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.SortOrder;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.common.data.plugin.ComponentDescriptor;
import org.thingsboard.server.common.data.plugin.ComponentType;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.rpc.Rpc;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
@ -103,11 +104,12 @@ import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
import org.thingsboard.server.dao.oauth2.OAuth2Service;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rpc.RpcService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.dao.tenant.TenantProfileService;
@ -122,11 +124,10 @@ import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.action.RuleEngineEntityActionService;
import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.edge.rpc.EdgeRpcService;
import org.thingsboard.server.service.ota.OtaPackageStateService;
import org.thingsboard.server.service.edge.EdgeNotificationService;
import org.thingsboard.server.service.edge.rpc.EdgeGrpcService;
import org.thingsboard.server.service.edge.rpc.EdgeRpcService;
import org.thingsboard.server.service.lwm2m.LwM2MServerSecurityInfoRepository;
import org.thingsboard.server.service.ota.OtaPackageStateService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.resource.TbResourceService;
@ -251,6 +252,9 @@ public abstract class BaseController {
@Autowired
protected OtaPackageStateService otaPackageStateService;
@Autowired
protected RpcService rpcService;
@Autowired
protected TbQueueProducerProvider producerProvider;
@ -789,6 +793,18 @@ public abstract class BaseController {
}
}
Rpc checkRpcId(RpcId rpcId, Operation operation) throws ThingsboardException {
try {
validateId(rpcId, "Incorrect rpcId " + rpcId);
Rpc rpc = rpcService.findById(getCurrentUser().getTenantId(), rpcId);
checkNotNull(rpc);
accessControlService.checkPermission(getCurrentUser(), Resource.RPC, operation, rpcId, rpc);
return rpc;
} catch (Exception e) {
throw handleException(e, false);
}
}
@SuppressWarnings("unchecked")
protected <I extends EntityId> I emptyId(EntityType entityType) {
return (I) EntityIdFactory.getByTypeAndUuid(entityType, ModelConstants.NULL_UUID);
@ -905,18 +921,12 @@ public abstract class BaseController {
if (EntityType.EDGE.equals(entityId.getEntityType())) {
return Collections.singletonList(new EdgeId(entityId.getId()));
}
PageDataIterableByTenantIdEntityId<EdgeId> relatedEdgeIdsIterator =
new PageDataIterableByTenantIdEntityId<>(edgeService::findRelatedEdgeIdsByEntityId, tenantId, entityId, DEFAULT_PAGE_SIZE);
List<EdgeId> result = new ArrayList<>();
PageLink pageLink = new PageLink(DEFAULT_PAGE_SIZE);
PageData<EdgeId> pageData;
do {
pageData = edgeService.findRelatedEdgeIdsByEntityId(tenantId, entityId, pageLink);
if (pageData != null && pageData.getData() != null && !pageData.getData().isEmpty()) {
result.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageLink.nextPageLink();
}
}
} while (pageData != null && pageData.hasNext());
for(EdgeId edgeId : relatedEdgeIdsIterator) {
result.add(edgeId);
}
return result;
}

4
application/src/main/java/org/thingsboard/server/controller/DashboardController.java

@ -657,7 +657,7 @@ public class DashboardController extends BaseController {
Edge edge = checkEdgeId(edgeId, Operation.READ);
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
checkDashboardId(dashboardId, Operation.ASSIGN_TO_EDGE);
checkDashboardId(dashboardId, Operation.READ);
Dashboard savedDashboard = checkNotNull(dashboardService.assignDashboardToEdge(getCurrentUser().getTenantId(), dashboardId, edgeId));
@ -689,7 +689,7 @@ public class DashboardController extends BaseController {
EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
Edge edge = checkEdgeId(edgeId, Operation.READ);
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.UNASSIGN_FROM_EDGE);
Dashboard dashboard = checkDashboardId(dashboardId, Operation.READ);
Dashboard savedDashboard = checkNotNull(dashboardService.unassignDashboardFromEdge(getCurrentUser().getTenantId(), dashboardId, edgeId));

4
application/src/main/java/org/thingsboard/server/controller/DeviceController.java

@ -682,7 +682,7 @@ public class DeviceController extends BaseController {
Edge edge = checkEdgeId(edgeId, Operation.READ);
DeviceId deviceId = new DeviceId(toUUID(strDeviceId));
checkDeviceId(deviceId, Operation.ASSIGN_TO_EDGE);
checkDeviceId(deviceId, Operation.READ);
Device savedDevice = checkNotNull(deviceService.assignDeviceToEdge(getCurrentUser().getTenantId(), deviceId, edgeId));
@ -716,7 +716,7 @@ public class DeviceController extends BaseController {
Edge edge = checkEdgeId(edgeId, Operation.READ);
DeviceId deviceId = new DeviceId(toUUID(strDeviceId));
Device device = checkDeviceId(deviceId, Operation.UNASSIGN_FROM_EDGE);
Device device = checkDeviceId(deviceId, Operation.READ);
Device savedDevice = checkNotNull(deviceService.unassignDeviceFromEdge(getCurrentUser().getTenantId(), deviceId, edgeId));

20
application/src/main/java/org/thingsboard/server/controller/EdgeController.java

@ -80,7 +80,7 @@ public class EdgeController extends BaseController {
EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
Edge edge = checkEdgeId(edgeId, Operation.READ);
if (Authority.CUSTOMER_USER.equals(getCurrentUser().getAuthority())) {
cleanUpSensitiveData(edge);
cleanUpLicenseKey(edge);
}
return edge;
} catch (Exception e) {
@ -97,7 +97,7 @@ public class EdgeController extends BaseController {
EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
EdgeInfo edgeInfo = checkEdgeInfoId(edgeId, Operation.READ);
if (Authority.CUSTOMER_USER.equals(getCurrentUser().getAuthority())) {
cleanUpSensitiveData(edgeInfo);
cleanUpLicenseKey(edgeInfo);
}
return edgeInfo;
} catch (Exception e) {
@ -127,7 +127,7 @@ public class EdgeController extends BaseController {
accessControlService.checkPermission(getCurrentUser(), Resource.EDGE, operation,
edge.getId(), edge);
Edge savedEdge = checkNotNull(edgeService.saveEdge(edge));
Edge savedEdge = checkNotNull(edgeService.saveEdge(edge, true));
if (created) {
ruleChainService.assignRuleChainToEdge(tenantId, edgeTemplateRootRuleChain.getId(), savedEdge.getId());
@ -404,7 +404,7 @@ public class EdgeController extends BaseController {
}
if (Authority.CUSTOMER_USER.equals(user.getAuthority())) {
for (Edge edge : result.getData()) {
cleanUpSensitiveData(edge);
cleanUpLicenseKey(edge);
}
}
return checkNotNull(result);
@ -439,7 +439,7 @@ public class EdgeController extends BaseController {
}
if (Authority.CUSTOMER_USER.equals(user.getAuthority())) {
for (Edge edge : result.getData()) {
cleanUpSensitiveData(edge);
cleanUpLicenseKey(edge);
}
}
return checkNotNull(result);
@ -471,7 +471,7 @@ public class EdgeController extends BaseController {
List<Edge> edges = edgesFuture.get();
if (Authority.CUSTOMER_USER.equals(user.getAuthority())) {
for (Edge edge : edges) {
cleanUpSensitiveData(edge);
cleanUpLicenseKey(edge);
}
}
return checkNotNull(edges);
@ -502,7 +502,7 @@ public class EdgeController extends BaseController {
}).collect(Collectors.toList());
if (Authority.CUSTOMER_USER.equals(user.getAuthority())) {
for (Edge edge : edges) {
cleanUpSensitiveData(edge);
cleanUpLicenseKey(edge);
}
}
return edges;
@ -580,11 +580,7 @@ public class EdgeController extends BaseController {
}
}
private void cleanUpSensitiveData(Edge edge) {
private void cleanUpLicenseKey(Edge edge) {
edge.setEdgeLicenseKey(null);
edge.setRoutingKey(null);
edge.setSecret(null);
edge.setCloudEndpoint(null);
edge.setRootRuleChainId(null);
}
}

4
application/src/main/java/org/thingsboard/server/controller/EntityViewController.java

@ -626,7 +626,7 @@ public class EntityViewController extends BaseController {
Edge edge = checkEdgeId(edgeId, Operation.READ);
EntityViewId entityViewId = new EntityViewId(toUUID(strEntityViewId));
checkEntityViewId(entityViewId, Operation.ASSIGN_TO_EDGE);
checkEntityViewId(entityViewId, Operation.READ);
EntityView savedEntityView = checkNotNull(entityViewService.assignEntityViewToEdge(getTenantId(), entityViewId, edgeId));
logEntityAction(entityViewId, savedEntityView,
@ -656,7 +656,7 @@ public class EntityViewController extends BaseController {
Edge edge = checkEdgeId(edgeId, Operation.READ);
EntityViewId entityViewId = new EntityViewId(toUUID(strEntityViewId));
EntityView entityView = checkEntityViewId(entityViewId, Operation.UNASSIGN_FROM_EDGE);
EntityView entityView = checkEntityViewId(entityViewId, Operation.READ);
EntityView savedEntityView = checkNotNull(entityViewService.unassignEntityViewFromEdge(getTenantId(), entityViewId, edgeId));
logEntityAction(entityViewId, entityView,

9
application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java

@ -32,12 +32,13 @@ import org.springframework.web.multipart.MultipartFile;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.ota.ChecksumAlgorithm;
import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.ota.ChecksumAlgorithm;
import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.queue.util.TbCoreComponent;
@ -109,12 +110,12 @@ public class OtaPackageController extends BaseController {
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/otaPackage", method = RequestMethod.POST)
@ResponseBody
public OtaPackageInfo saveOtaPackageInfo(@RequestBody OtaPackageInfo otaPackageInfo) throws ThingsboardException {
public OtaPackageInfo saveOtaPackageInfo(@RequestBody SaveOtaPackageInfoRequest otaPackageInfo) throws ThingsboardException {
boolean created = otaPackageInfo.getId() == null;
try {
otaPackageInfo.setTenantId(getTenantId());
checkEntity(otaPackageInfo.getId(), otaPackageInfo, Resource.OTA_PACKAGE);
OtaPackageInfo savedOtaPackageInfo = otaPackageService.saveOtaPackageInfo(otaPackageInfo);
OtaPackageInfo savedOtaPackageInfo = otaPackageService.saveOtaPackageInfo(new OtaPackageInfo(otaPackageInfo), otaPackageInfo.isUsesUrl());
logEntityAction(savedOtaPackageInfo.getId(), savedOtaPackageInfo,
null, created ? ActionType.ADDED : ActionType.UPDATED, null);
return savedOtaPackageInfo;

57
application/src/main/java/org/thingsboard/server/controller/RpcController.java

@ -29,17 +29,24 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.rule.engine.api.RpcError;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RpcId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UUIDBased;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rpc.Rpc;
import org.thingsboard.server.common.data.rpc.RpcStatus;
import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody;
import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest;
import org.thingsboard.server.queue.util.TbCoreComponent;
@ -93,6 +100,52 @@ public class RpcController extends BaseController {
return handleDeviceRPCRequest(false, new DeviceId(UUID.fromString(deviceIdStr)), requestBody);
}
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/persistent/{rpcId}", method = RequestMethod.GET)
@ResponseBody
public Rpc getPersistedRpc(@PathVariable("rpcId") String strRpc) throws ThingsboardException {
checkParameter("RpcId", strRpc);
try {
RpcId rpcId = new RpcId(UUID.fromString(strRpc));
return checkRpcId(rpcId, Operation.READ);
} catch (Exception e) {
throw handleException(e);
}
}
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/persistent/device/{deviceId}", method = RequestMethod.GET)
@ResponseBody
public PageData<Rpc> getPersistedRpcByDevice(@PathVariable("deviceId") String strDeviceId,
@RequestParam int pageSize,
@RequestParam int page,
@RequestParam RpcStatus rpcStatus,
@RequestParam(required = false) String textSearch,
@RequestParam(required = false) String sortProperty,
@RequestParam(required = false) String sortOrder) throws ThingsboardException {
checkParameter("DeviceId", strDeviceId);
try {
TenantId tenantId = getCurrentUser().getTenantId();
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder);
DeviceId deviceId = new DeviceId(UUID.fromString(strDeviceId));
return checkNotNull(rpcService.findAllByDeviceIdAndStatus(tenantId, deviceId, rpcStatus, pageLink));
} catch (Exception e) {
throw handleException(e);
}
}
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/persistent/{rpcId}", method = RequestMethod.DELETE)
@ResponseBody
public void deleteResource(@PathVariable("rpcId") String strRpc) throws ThingsboardException {
checkParameter("RpcId", strRpc);
try {
rpcService.deleteRpc(getTenantId(), new RpcId(UUID.fromString(strRpc)));
} catch (Exception e) {
throw handleException(e);
}
}
private DeferredResult<ResponseEntity> handleDeviceRPCRequest(boolean oneWay, DeviceId deviceId, String requestBody) throws ThingsboardException {
try {
JsonNode rpcRequestBody = jsonMapper.readTree(requestBody);
@ -103,6 +156,7 @@ public class RpcController extends BaseController {
long timeout = rpcRequestBody.has("timeout") ? rpcRequestBody.get("timeout").asLong() : defaultTimeout;
long expTime = System.currentTimeMillis() + Math.max(minTimeout, timeout);
UUID rpcRequestUUID = rpcRequestBody.has("requestUUID") ? UUID.fromString(rpcRequestBody.get("requestUUID").asText()) : UUID.randomUUID();
boolean persisted = rpcRequestBody.has(DataConstants.PERSISTENT) && rpcRequestBody.get(DataConstants.PERSISTENT).asBoolean();
accessValidator.validate(currentUser, Operation.RPC_CALL, deviceId, new HttpValidationCallback(response, new FutureCallback<DeferredResult<ResponseEntity>>() {
@Override
public void onSuccess(@Nullable DeferredResult<ResponseEntity> result) {
@ -111,7 +165,8 @@ public class RpcController extends BaseController {
deviceId,
oneWay,
expTime,
body
body,
persisted
);
deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse), currentUser);
}

21
application/src/main/java/org/thingsboard/server/controller/RuleChainController.java

@ -50,6 +50,7 @@ import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageDataIterableByTenant;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.rule.DefaultRuleChainCreateRequest;
@ -505,7 +506,7 @@ public class RuleChainController extends BaseController {
Edge edge = checkEdgeId(edgeId, Operation.READ);
RuleChainId ruleChainId = new RuleChainId(toUUID(strRuleChainId));
checkRuleChain(ruleChainId, Operation.ASSIGN_TO_EDGE);
checkRuleChain(ruleChainId, Operation.READ);
RuleChain savedRuleChain = checkNotNull(ruleChainService.assignRuleChainToEdge(getCurrentUser().getTenantId(), ruleChainId, edgeId));
@ -537,7 +538,7 @@ public class RuleChainController extends BaseController {
EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
Edge edge = checkEdgeId(edgeId, Operation.READ);
RuleChainId ruleChainId = new RuleChainId(toUUID(strRuleChainId));
RuleChain ruleChain = checkRuleChain(ruleChainId, Operation.UNASSIGN_FROM_EDGE);
RuleChain ruleChain = checkRuleChain(ruleChainId, Operation.READ);
RuleChain savedRuleChain = checkNotNull(ruleChainService.unassignRuleChainFromEdge(getCurrentUser().getTenantId(), ruleChainId, edgeId, false));
@ -645,17 +646,11 @@ public class RuleChainController extends BaseController {
try {
TenantId tenantId = getCurrentUser().getTenantId();
List<RuleChain> result = new ArrayList<>();
PageLink pageLink = new PageLink(DEFAULT_PAGE_SIZE);
PageData<RuleChain> pageData;
do {
pageData = ruleChainService.findAutoAssignToEdgeRuleChainsByTenantId(tenantId, pageLink);
if (pageData.getData().size() > 0) {
result.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageLink.nextPageLink();
}
}
} while (pageData.hasNext());
PageDataIterableByTenant<RuleChain> autoAssignRuleChainsIterator =
new PageDataIterableByTenant<>(ruleChainService::findAutoAssignToEdgeRuleChainsByTenantId, tenantId, DEFAULT_PAGE_SIZE);
for (RuleChain ruleChain : autoAssignRuleChainsIterator) {
result.add(ruleChain);
}
return checkNotNull(result);
} catch (Exception e) {
throw handleException(e);

15
application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java

@ -25,9 +25,12 @@ import org.springframework.security.authentication.BadCredentialsException;
import org.springframework.security.authentication.DisabledException;
import org.springframework.security.authentication.LockedException;
import org.springframework.security.core.AuthenticationException;
import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.security.web.access.AccessDeniedHandler;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
@ -40,14 +43,15 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
@Component
@Slf4j
public class ThingsboardErrorResponseHandler implements AccessDeniedHandler {
@RestControllerAdvice
public class ThingsboardErrorResponseHandler extends ResponseEntityExceptionHandler implements AccessDeniedHandler {
@Autowired
private ObjectMapper mapper;
@Override
@ExceptionHandler(AccessDeniedException.class)
public void handle(HttpServletRequest request, HttpServletResponse response,
AccessDeniedException accessDeniedException) throws IOException,
ServletException {
@ -60,6 +64,7 @@ public class ThingsboardErrorResponseHandler implements AccessDeniedHandler {
}
}
@ExceptionHandler(Exception.class)
public void handle(Exception exception, HttpServletResponse response) {
log.debug("Processing exception {}", exception.getMessage(), exception);
if (!response.isCommitted()) {
@ -148,7 +153,7 @@ public class ThingsboardErrorResponseHandler implements AccessDeniedHandler {
private void handleAuthenticationException(AuthenticationException authenticationException, HttpServletResponse response) throws IOException {
response.setStatus(HttpStatus.UNAUTHORIZED.value());
if (authenticationException instanceof BadCredentialsException) {
if (authenticationException instanceof BadCredentialsException || authenticationException instanceof UsernameNotFoundException) {
mapper.writeValue(response.getWriter(), ThingsboardErrorResponse.of("Invalid username or password", ThingsboardErrorCode.AUTHENTICATION, HttpStatus.UNAUTHORIZED));
} else if (authenticationException instanceof DisabledException) {
mapper.writeValue(response.getWriter(), ThingsboardErrorResponse.of("User account is not active", ThingsboardErrorCode.AUTHENTICATION, HttpStatus.UNAUTHORIZED));
@ -159,7 +164,7 @@ public class ThingsboardErrorResponseHandler implements AccessDeniedHandler {
} else if (authenticationException instanceof AuthMethodNotSupportedException) {
mapper.writeValue(response.getWriter(), ThingsboardErrorResponse.of(authenticationException.getMessage(), ThingsboardErrorCode.AUTHENTICATION, HttpStatus.UNAUTHORIZED));
} else if (authenticationException instanceof UserPasswordExpiredException) {
UserPasswordExpiredException expiredException = (UserPasswordExpiredException)authenticationException;
UserPasswordExpiredException expiredException = (UserPasswordExpiredException) authenticationException;
String resetToken = expiredException.getResetToken();
mapper.writeValue(response.getWriter(), ThingsboardCredentialsExpiredResponse.of(expiredException.getMessage(), resetToken));
} else {

2
application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeNotificationService.java

@ -100,7 +100,7 @@ public class DefaultEdgeNotificationService implements EdgeNotificationService {
@Override
public Edge setEdgeRootRuleChain(TenantId tenantId, Edge edge, RuleChainId ruleChainId) throws IOException {
edge.setRootRuleChainId(ruleChainId);
Edge savedEdge = edgeService.saveEdge(edge);
Edge savedEdge = edgeService.saveEdge(edge, true);
saveEdgeEvent(tenantId, edge.getId(), EdgeEventType.RULE_CHAIN, EdgeEventActionType.UPDATED, ruleChainId, null);
return savedEdge;
}

34
application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java

@ -16,7 +16,6 @@
package org.thingsboard.server.service.edge;
import lombok.Data;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
@ -49,121 +48,98 @@ import org.thingsboard.server.service.edge.rpc.processor.WidgetBundleEdgeProcess
import org.thingsboard.server.service.edge.rpc.processor.WidgetTypeEdgeProcessor;
import org.thingsboard.server.service.edge.rpc.sync.EdgeRequestsService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.executors.GrpcCallbackExecutorService;
@Component
@TbCoreComponent
@Data
@Lazy
public class EdgeContextComponent {
@Lazy
@Autowired
private EdgeService edgeService;
@Lazy
@Autowired
private EdgeEventService edgeEventService;
@Lazy
@Autowired
private AdminSettingsService adminSettingsService;
@Lazy
@Autowired
private AssetService assetService;
@Lazy
@Autowired
private DeviceProfileService deviceProfileService;
@Lazy
@Autowired
private AttributesService attributesService;
@Lazy
@Autowired
private DashboardService dashboardService;
@Lazy
@Autowired
private RuleChainService ruleChainService;
@Lazy
@Autowired
private UserService userService;
@Lazy
@Autowired
private WidgetsBundleService widgetsBundleService;
@Lazy
@Autowired
private EdgeRequestsService edgeRequestsService;
@Lazy
@Autowired
private AlarmEdgeProcessor alarmProcessor;
@Lazy
@Autowired
private DeviceProfileEdgeProcessor deviceProfileProcessor;
@Lazy
@Autowired
private DeviceEdgeProcessor deviceProcessor;
@Lazy
@Autowired
private EntityEdgeProcessor entityProcessor;
@Lazy
@Autowired
private AssetEdgeProcessor assetProcessor;
@Lazy
@Autowired
private EntityViewEdgeProcessor entityViewProcessor;
@Lazy
@Autowired
private UserEdgeProcessor userProcessor;
@Lazy
@Autowired
private RelationEdgeProcessor relationProcessor;
@Lazy
@Autowired
private TelemetryEdgeProcessor telemetryProcessor;
@Lazy
@Autowired
private DashboardEdgeProcessor dashboardProcessor;
@Lazy
@Autowired
private RuleChainEdgeProcessor ruleChainProcessor;
@Lazy
@Autowired
private CustomerEdgeProcessor customerProcessor;
@Lazy
@Autowired
private WidgetBundleEdgeProcessor widgetBundleProcessor;
@Lazy
@Autowired
private WidgetTypeEdgeProcessor widgetTypeProcessor;
@Lazy
@Autowired
private AdminSettingsEdgeProcessor adminSettingsProcessor;
@Lazy
@Autowired
private EdgeEventStorageSettings edgeEventStorageSettings;
@Autowired
@Getter
private DbCallbackExecutorService dbCallbackExecutor;
@Autowired
private GrpcCallbackExecutorService grpcCallbackExecutorService;
}

111
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java

@ -16,8 +16,8 @@
package org.thingsboard.server.service.edge.rpc;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.io.Resources;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import io.grpc.Server;
import io.grpc.netty.NettyServerBuilder;
import io.grpc.stub.StreamObserver;
@ -28,6 +28,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.ResourceUtils;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.TenantId;
@ -45,18 +46,20 @@ import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@Service
@Slf4j
@ -65,7 +68,8 @@ import java.util.concurrent.TimeUnit;
public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase implements EdgeRpcService {
private final ConcurrentMap<EdgeId, EdgeGrpcSession> sessions = new ConcurrentHashMap<>();
private final ConcurrentMap<EdgeId, Boolean> sessionNewEvents = new ConcurrentHashMap<>();
private final ConcurrentMap<EdgeId, Lock> sessionNewEventsLocks = new ConcurrentHashMap<>();
private final Map<EdgeId, Boolean> sessionNewEvents = new HashMap<>();
private final ConcurrentMap<EdgeId, ScheduledFuture<?>> sessionEdgeEventChecks = new ConcurrentHashMap<>();
private static final ObjectMapper mapper = new ObjectMapper();
@ -81,10 +85,15 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
private boolean persistToTelemetry;
@Value("${edges.rpc.client_max_keep_alive_time_sec}")
private int clientMaxKeepAliveTimeSec;
@Value("${edges.rpc.max_inbound_message_size:4194304}")
private int maxInboundMessageSize;
@Value("${edges.scheduler_pool_size}")
private int schedulerPoolSize;
@Value("${edges.send_scheduler_pool_size}")
private int sendSchedulerPoolSize;
@Autowired
private EdgeContextComponent ctx;
@ -93,21 +102,22 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
private Server server;
private ScheduledExecutorService scheduler;
private ScheduledExecutorService edgeEventProcessingExecutorService;
private ExecutorService syncExecutorService;
private ScheduledExecutorService sendDownlinkExecutorService;
@PostConstruct
public void init() {
log.info("Initializing Edge RPC service!");
NettyServerBuilder builder = NettyServerBuilder.forPort(rpcPort)
.permitKeepAliveTime(clientMaxKeepAliveTimeSec, TimeUnit.SECONDS)
.maxInboundMessageSize(maxInboundMessageSize)
.addService(this);
if (sslEnabled) {
try {
File certFile = new File(Resources.getResource(certFileResource).toURI());
File privateKeyFile = new File(Resources.getResource(privateKeyResource).toURI());
builder.useTransportSecurity(certFile, privateKeyFile);
InputStream certFileIs = ResourceUtils.getInputStream(this, certFileResource);
InputStream privateKeyFileIs = ResourceUtils.getInputStream(this, privateKeyResource);
builder.useTransportSecurity(certFileIs, privateKeyFileIs);
} catch (Exception e) {
log.error("Unable to set up SSL context. Reason: " + e.getMessage(), e);
throw new RuntimeException("Unable to set up SSL context!", e);
@ -121,9 +131,8 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
log.error("Failed to start Edge RPC server!", e);
throw new RuntimeException("Failed to start Edge RPC server!");
}
this.scheduler = Executors.newScheduledThreadPool(schedulerPoolSize, ThingsBoardThreadFactory.forName("edge-scheduler"));
this.syncExecutorService = Executors.newFixedThreadPool(
Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("edge-sync"));
this.edgeEventProcessingExecutorService = Executors.newScheduledThreadPool(schedulerPoolSize, ThingsBoardThreadFactory.forName("edge-scheduler"));
this.sendDownlinkExecutorService = Executors.newScheduledThreadPool(sendSchedulerPoolSize, ThingsBoardThreadFactory.forName("edge-send-scheduler"));
log.info("Edge RPC service initialized!");
}
@ -140,17 +149,17 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
sessionEdgeEventChecks.remove(edgeId);
}
}
if (scheduler != null) {
scheduler.shutdownNow();
if (edgeEventProcessingExecutorService != null) {
edgeEventProcessingExecutorService.shutdownNow();
}
if (syncExecutorService != null) {
syncExecutorService.shutdownNow();
if (sendDownlinkExecutorService != null) {
sendDownlinkExecutorService.shutdownNow();
}
}
@Override
public StreamObserver<RequestMsg> handleMsgs(StreamObserver<ResponseMsg> outputStream) {
return new EdgeGrpcSession(ctx, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, mapper, syncExecutorService).getInputStream();
return new EdgeGrpcSession(ctx, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, mapper, sendDownlinkExecutorService).getInputStream();
}
@Override
@ -171,7 +180,13 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
log.info("[{}] Closing and removing session for edge [{}]", tenantId, edgeId);
session.close();
sessions.remove(edgeId);
sessionNewEvents.remove(edgeId);
final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock());
newEventLock.lock();
try {
sessionNewEvents.remove(edgeId);
} finally {
newEventLock.unlock();
}
cancelScheduleEdgeEventsCheck(edgeId);
}
}
@ -179,16 +194,28 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
@Override
public void onEdgeEvent(TenantId tenantId, EdgeId edgeId) {
log.trace("[{}] onEdgeEvent [{}]", tenantId, edgeId.getId());
if (Boolean.FALSE.equals(sessionNewEvents.get(edgeId))) {
log.trace("[{}] set session new events flag to true [{}]", tenantId, edgeId.getId());
sessionNewEvents.put(edgeId, true);
final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock());
newEventLock.lock();
try {
if (Boolean.FALSE.equals(sessionNewEvents.get(edgeId))) {
log.trace("[{}] set session new events flag to true [{}]", tenantId, edgeId.getId());
sessionNewEvents.put(edgeId, true);
}
} finally {
newEventLock.unlock();
}
}
private void onEdgeConnect(EdgeId edgeId, EdgeGrpcSession edgeGrpcSession) {
log.info("[{}] edge [{}] connected successfully.", edgeGrpcSession.getSessionId(), edgeId);
sessions.put(edgeId, edgeGrpcSession);
sessionNewEvents.put(edgeId, true);
final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock());
newEventLock.lock();
try {
sessionNewEvents.put(edgeId, true);
} finally {
newEventLock.unlock();
}
save(edgeId, DefaultDeviceStateService.ACTIVITY_STATE, true);
save(edgeId, DefaultDeviceStateService.LAST_CONNECT_TIME, System.currentTimeMillis());
cancelScheduleEdgeEventsCheck(edgeId);
@ -210,19 +237,37 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
EdgeId edgeId = session.getEdge().getId();
UUID tenantId = session.getEdge().getTenantId().getId();
if (sessions.containsKey(edgeId)) {
ScheduledFuture<?> schedule = scheduler.schedule(() -> {
ScheduledFuture<?> edgeEventCheckTask = edgeEventProcessingExecutorService.schedule(() -> {
try {
if (Boolean.TRUE.equals(sessionNewEvents.get(edgeId))) {
log.trace("[{}] Set session new events flag to false", edgeId.getId());
sessionNewEvents.put(edgeId, false);
session.processEdgeEvents();
final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock());
newEventLock.lock();
try {
if (Boolean.TRUE.equals(sessionNewEvents.get(edgeId))) {
log.trace("[{}] Set session new events flag to false", edgeId.getId());
sessionNewEvents.put(edgeId, false);
Futures.addCallback(session.processEdgeEvents(), new FutureCallback<>() {
@Override
public void onSuccess(Void result) {
scheduleEdgeEventsCheck(session);
}
@Override
public void onFailure(Throwable t) {
log.warn("[{}] Failed to process edge events for edge [{}]!", tenantId, session.getEdge().getId().getId(), t);
scheduleEdgeEventsCheck(session);
}
}, ctx.getGrpcCallbackExecutorService());
} else {
scheduleEdgeEventsCheck(session);
}
} finally {
newEventLock.unlock();
}
} catch (Exception e) {
log.warn("[{}] Failed to process edge events for edge [{}]!", tenantId, session.getEdge().getId().getId(), e);
}
scheduleEdgeEventsCheck(session);
}, ctx.getEdgeEventStorageSettings().getNoRecordsSleepInterval(), TimeUnit.MILLISECONDS);
sessionEdgeEventChecks.put(edgeId, schedule);
sessionEdgeEventChecks.put(edgeId, edgeEventCheckTask);
log.trace("[{}] Check edge event scheduled for edge [{}]", tenantId, edgeId.getId());
} else {
log.debug("[{}] Session was removed and edge event check schedule must not be started [{}]",
@ -244,7 +289,13 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
private void onEdgeDisconnect(EdgeId edgeId) {
log.info("[{}] edge disconnected!", edgeId);
sessions.remove(edgeId);
sessionNewEvents.remove(edgeId);
final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock());
newEventLock.lock();
try {
sessionNewEvents.remove(edgeId);
} finally {
newEventLock.unlock();
}
save(edgeId, DefaultDeviceStateService.ACTIVITY_STATE, false);
save(edgeId, DefaultDeviceStateService.LAST_DISCONNECT_TIME, System.currentTimeMillis());
cancelScheduleEdgeEventsCheck(edgeId);

300
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java

@ -20,7 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import io.grpc.stub.StreamObserver;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@ -31,9 +31,7 @@ import org.thingsboard.server.common.data.EdgeUtils;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
@ -69,32 +67,20 @@ import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg;
import org.thingsboard.server.gen.edge.v1.UserCredentialsRequestMsg;
import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg;
import org.thingsboard.server.service.edge.EdgeContextComponent;
import org.thingsboard.server.service.edge.rpc.fetch.AdminSettingsEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.AssetsEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.CustomerUsersEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.DashboardsEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.DeviceProfilesEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.EdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.GeneralEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.RuleChainsEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.SystemWidgetsBundlesEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.TenantAdminUsersEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.TenantWidgetsBundlesEdgeEventFetcher;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.BiConsumer;
@ -106,7 +92,6 @@ import java.util.stream.Collectors;
public final class EdgeGrpcSession implements Closeable {
private static final ReentrantLock downlinkMsgLock = new ReentrantLock();
private static final ReentrantLock downlinkMsgsPackLock = new ReentrantLock();
private static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs";
@ -115,7 +100,7 @@ public final class EdgeGrpcSession implements Closeable {
private final Consumer<EdgeId> sessionCloseListener;
private final ObjectMapper mapper;
private final Map<Integer, DownlinkMsg> pendingMsgsMap = new LinkedHashMap<>();
private final EdgeSessionState sessionState = new EdgeSessionState();
private EdgeContextComponent ctx;
private Edge edge;
@ -124,19 +109,17 @@ public final class EdgeGrpcSession implements Closeable {
private boolean connected;
private boolean syncCompleted;
private ExecutorService syncExecutorService;
private CountDownLatch latch;
private ScheduledExecutorService sendDownlinkExecutorService;
EdgeGrpcSession(EdgeContextComponent ctx, StreamObserver<ResponseMsg> outputStream, BiConsumer<EdgeId, EdgeGrpcSession> sessionOpenListener,
Consumer<EdgeId> sessionCloseListener, ObjectMapper mapper, ExecutorService syncExecutorService) {
Consumer<EdgeId> sessionCloseListener, ObjectMapper mapper, ScheduledExecutorService sendDownlinkExecutorService) {
this.sessionId = UUID.randomUUID();
this.ctx = ctx;
this.outputStream = outputStream;
this.sessionOpenListener = sessionOpenListener;
this.sessionCloseListener = sessionCloseListener;
this.mapper = mapper;
this.syncExecutorService = syncExecutorService;
this.sendDownlinkExecutorService = sendDownlinkExecutorService;
initInputStream();
}
@ -155,18 +138,21 @@ public final class EdgeGrpcSession implements Closeable {
connected = true;
}
}
if (connected && requestMsg.getMsgType().equals(RequestMsgType.SYNC_REQUEST_RPC_MESSAGE)) {
if (requestMsg.getSyncRequestMsg().getSyncRequired()) {
startSyncProcess(edge.getTenantId(), edge.getId());
}
syncCompleted = true;
}
if (connected) {
if (requestMsg.getMsgType().equals(RequestMsgType.UPLINK_RPC_MESSAGE) && requestMsg.hasUplinkMsg()) {
onUplinkMsg(requestMsg.getUplinkMsg());
if (requestMsg.getMsgType().equals(RequestMsgType.SYNC_REQUEST_RPC_MESSAGE)) {
if (requestMsg.hasSyncRequestMsg() && requestMsg.getSyncRequestMsg().getSyncRequired()) {
startSyncProcess(edge.getTenantId(), edge.getId());
} else {
syncCompleted = true;
}
}
if (requestMsg.getMsgType().equals(RequestMsgType.UPLINK_RPC_MESSAGE) && requestMsg.hasDownlinkResponseMsg()) {
onDownlinkResponse(requestMsg.getDownlinkResponseMsg());
if (requestMsg.getMsgType().equals(RequestMsgType.UPLINK_RPC_MESSAGE)) {
if (requestMsg.hasUplinkMsg()) {
onUplinkMsg(requestMsg.getUplinkMsg());
}
if (requestMsg.hasDownlinkResponseMsg()) {
onDownlinkResponse(requestMsg.getDownlinkResponseMsg());
}
}
}
}
@ -201,37 +187,41 @@ public final class EdgeGrpcSession implements Closeable {
public void startSyncProcess(TenantId tenantId, EdgeId edgeId) {
log.trace("[{}][{}] Staring edge sync process", tenantId, edgeId);
syncCompleted = false;
syncExecutorService.submit(() -> {
try {
startProcessingEdgeEvents(new SystemWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService()));
startProcessingEdgeEvents(new TenantWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService()));
startProcessingEdgeEvents(new DeviceProfilesEdgeEventFetcher(ctx.getDeviceProfileService()));
startProcessingEdgeEvents(new RuleChainsEdgeEventFetcher(ctx.getRuleChainService()));
startProcessingEdgeEvents(new TenantAdminUsersEdgeEventFetcher(ctx.getUserService()));
if (edge.getCustomerId() != null && !EntityId.NULL_UUID.equals(edge.getCustomerId().getId())) {
EdgeEvent customerEdgeEvent = EdgeEventUtils.constructEdgeEvent(edge.getTenantId(), edge.getId(),
EdgeEventType.CUSTOMER, EdgeEventActionType.ADDED, edge.getCustomerId(), null);
DownlinkMsg customerDownlinkMsg = convertToDownlinkMsg(customerEdgeEvent);
sendDownlinkMsgsPack(Collections.singletonList(customerDownlinkMsg));
startProcessingEdgeEvents(new CustomerUsersEdgeEventFetcher(ctx.getUserService(), edge.getCustomerId()));
}
doSync(new EdgeSyncCursor(ctx, edge));
}
startProcessingEdgeEvents(new AdminSettingsEdgeEventFetcher(ctx.getAdminSettingsService()));
private void doSync(EdgeSyncCursor cursor) {
if (cursor.hasNext()) {
log.info("[{}][{}] starting sync process, cursor current idx = {}", edge.getTenantId(), edge.getId(), cursor.getCurrentIdx());
ListenableFuture<UUID> uuidListenableFuture = startProcessingEdgeEvents(cursor.getNext());
Futures.addCallback(uuidListenableFuture, new FutureCallback<>() {
@Override
public void onSuccess(@Nullable UUID result) {
doSync(cursor);
}
startProcessingEdgeEvents(new AssetsEdgeEventFetcher(ctx.getAssetService()));
startProcessingEdgeEvents(new DashboardsEdgeEventFetcher(ctx.getDashboardService()));
@Override
public void onFailure(Throwable t) {
log.error("[{}][{}] Exception during sync process", edge.getTenantId(), edge.getId(), t);
}
}, ctx.getGrpcCallbackExecutorService());
} else {
DownlinkMsg syncCompleteDownlinkMsg = DownlinkMsg.newBuilder()
.setDownlinkMsgId(EdgeUtils.nextPositiveInt())
.setSyncCompletedMsg(SyncCompletedMsg.newBuilder().build())
.build();
Futures.addCallback(sendDownlinkMsgsPack(Collections.singletonList(syncCompleteDownlinkMsg)), new FutureCallback<Void>() {
@Override
public void onSuccess(Void result) {
syncCompleted = true;
}
DownlinkMsg syncCompleteDownlinkMsg = DownlinkMsg.newBuilder()
.setDownlinkMsgId(EdgeUtils.nextPositiveInt())
.setSyncCompletedMsg(SyncCompletedMsg.newBuilder().build())
.build();
sendDownlinkMsgsPack(Collections.singletonList(syncCompleteDownlinkMsg));
} catch (Exception e) {
log.error("[{}][{}] Exception during sync process", edge.getTenantId(), edge.getId(), e);
}
});
@Override
public void onFailure(Throwable t) {
log.error("[{}][{}] Exception during sending sync complete", edge.getTenantId(), edge.getId(), t);
}
}, ctx.getGrpcCallbackExecutorService());
}
}
private void onUplinkMsg(UplinkMsg uplinkMsg) {
@ -249,25 +239,32 @@ public final class EdgeGrpcSession implements Closeable {
@Override
public void onFailure(Throwable t) {
String errorMsg = t.getMessage() != null ? t.getMessage() : "";
UplinkResponseMsg uplinkResponseMsg = UplinkResponseMsg.newBuilder()
.setUplinkMsgId(uplinkMsg.getUplinkMsgId())
.setSuccess(false).setErrorMsg(t.getMessage()).build();
.setSuccess(false).setErrorMsg(errorMsg).build();
sendDownlinkMsg(ResponseMsg.newBuilder()
.setUplinkResponseMsg(uplinkResponseMsg)
.build());
}
}, MoreExecutors.directExecutor());
}, ctx.getGrpcCallbackExecutorService());
}
private void onDownlinkResponse(DownlinkResponseMsg msg) {
try {
if (msg.getSuccess()) {
pendingMsgsMap.remove(msg.getDownlinkMsgId());
sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId());
log.debug("[{}] Msg has been processed successfully! {}", edge.getRoutingKey(), msg);
} else {
log.error("[{}] Msg processing failed! Error msg: {}", edge.getRoutingKey(), msg.getErrorMsg());
}
latch.countDown();
if (sessionState.getPendingMsgsMap().isEmpty()) {
log.debug("[{}] Pending msgs map is empty. Stopping current iteration {}", edge.getRoutingKey(), msg);
if (sessionState.getScheduledSendDownlinkTask() != null) {
sessionState.getScheduledSendDownlinkTask().cancel(false);
}
sessionState.getSendDownlinkMsgsFuture().set(null);
}
} catch (Exception e) {
log.error("[{}] Can't process downlink response message [{}]", this.sessionId, msg, e);
}
@ -276,8 +273,8 @@ public final class EdgeGrpcSession implements Closeable {
private void sendDownlinkMsg(ResponseMsg downlinkMsg) {
log.trace("[{}] Sending downlink msg [{}]", this.sessionId, downlinkMsg);
if (isConnected()) {
downlinkMsgLock.lock();
try {
downlinkMsgLock.lock();
outputStream.onNext(downlinkMsg);
} catch (Exception e) {
log.error("[{}] Failed to send downlink message [{}]", this.sessionId, downlinkMsg, e);
@ -301,75 +298,140 @@ public final class EdgeGrpcSession implements Closeable {
sendDownlinkMsg(edgeConfigMsg);
}
void processEdgeEvents() throws Exception {
log.trace("[{}] processHandleMessages started", this.sessionId);
ListenableFuture<Void> processEdgeEvents() throws Exception {
SettableFuture<Void> result = SettableFuture.create();
log.trace("[{}] starting processing edge events", this.sessionId);
if (isConnected() && isSyncCompleted()) {
Long queueStartTs = getQueueStartTs().get();
GeneralEdgeEventFetcher fetcher = new GeneralEdgeEventFetcher(
queueStartTs,
ctx.getEdgeEventService());
UUID ifOffset = startProcessingEdgeEvents(fetcher);
if (ifOffset != null) {
Long newStartTs = Uuids.unixTimestamp(ifOffset);
updateQueueStartTs(newStartTs);
}
ListenableFuture<UUID> ifOffsetFuture = startProcessingEdgeEvents(fetcher);
Futures.addCallback(ifOffsetFuture, new FutureCallback<>() {
@Override
public void onSuccess(@Nullable UUID ifOffset) {
if (ifOffset != null) {
Long newStartTs = Uuids.unixTimestamp(ifOffset);
ListenableFuture<List<Void>> updateFuture = updateQueueStartTs(newStartTs);
Futures.addCallback(updateFuture, new FutureCallback<>() {
@Override
public void onSuccess(@Nullable List<Void> list) {
log.debug("[{}] queue offset was updated [{}][{}]", sessionId, ifOffset, newStartTs);
result.set(null);
}
@Override
public void onFailure(Throwable t) {
log.error("[{}] Failed to update queue offset [{}]", sessionId, ifOffset, t);
result.setException(t);
}
}, ctx.getGrpcCallbackExecutorService());
} else {
log.trace("[{}] ifOffset is null. Skipping iteration without db update", sessionId);
result.set(null);
}
}
@Override
public void onFailure(Throwable t) {
log.error("[{}] Failed to process events", sessionId, t);
result.setException(t);
}
}, ctx.getGrpcCallbackExecutorService());
} else {
log.trace("[{}] edge is not connected or sync is not completed. Skipping iteration", sessionId);
result.set(null);
}
log.trace("[{}] processHandleMessages finished", this.sessionId);
return result;
}
private UUID startProcessingEdgeEvents(EdgeEventFetcher fetcher) throws Exception {
private ListenableFuture<UUID> startProcessingEdgeEvents(EdgeEventFetcher fetcher) {
SettableFuture<UUID> result = SettableFuture.create();
PageLink pageLink = fetcher.getPageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount());
PageData<EdgeEvent> pageData;
UUID ifOffset = null;
boolean success;
do {
pageData = fetcher.fetchEdgeEvents(edge.getTenantId(), edge, pageLink);
processEdgeEvents(fetcher, pageLink, result);
return result;
}
private void processEdgeEvents(EdgeEventFetcher fetcher, PageLink pageLink, SettableFuture<UUID> result) {
try {
PageData<EdgeEvent> pageData = fetcher.fetchEdgeEvents(edge.getTenantId(), edge, pageLink);
if (isConnected() && !pageData.getData().isEmpty()) {
log.trace("[{}] [{}] event(s) are going to be processed.", this.sessionId, pageData.getData().size());
List<DownlinkMsg> downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData());
success = sendDownlinkMsgsPack(downlinkMsgsPack);
ifOffset = pageData.getData().get(pageData.getData().size() - 1).getUuidId();
if (success) {
pageLink = pageLink.nextPageLink();
}
Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void tmp) {
if (isConnected() && pageData.hasNext()) {
processEdgeEvents(fetcher, pageLink.nextPageLink(), result);
} else {
UUID ifOffset = pageData.getData().get(pageData.getData().size() - 1).getUuidId();
result.set(ifOffset);
}
}
@Override
public void onFailure(Throwable t) {
log.error("[{}] Failed to send downlink msgs pack", sessionId, t);
result.setException(t);
}
}, ctx.getGrpcCallbackExecutorService());
} else {
log.trace("[{}] no event(s) found. Stop processing edge events", this.sessionId);
result.set(null);
}
} while (isConnected() && pageData.hasNext());
return ifOffset;
} catch (Exception e) {
log.error("[{}] Failed to fetch edge events", this.sessionId, e);
result.setException(e);
}
}
private boolean sendDownlinkMsgsPack(List<DownlinkMsg> downlinkMsgsPack) throws InterruptedException {
try {
downlinkMsgsPackLock.lock();
boolean success;
pendingMsgsMap.clear();
downlinkMsgsPack.forEach(msg -> pendingMsgsMap.put(msg.getDownlinkMsgId(), msg));
do {
log.trace("[{}] [{}] downlink msg(s) are going to be send.", this.sessionId, pendingMsgsMap.values().size());
latch = new CountDownLatch(pendingMsgsMap.values().size());
List<DownlinkMsg> copy = new ArrayList<>(pendingMsgsMap.values());
for (DownlinkMsg downlinkMsg : copy) {
sendDownlinkMsg(ResponseMsg.newBuilder()
.setDownlinkMsg(downlinkMsg)
.build());
}
success = latch.await(10, TimeUnit.SECONDS);
if (!success || pendingMsgsMap.values().size() > 0) {
log.warn("[{}] Failed to deliver the batch: {}", this.sessionId, pendingMsgsMap.values());
}
if (isConnected() && (!success || pendingMsgsMap.values().size() > 0)) {
try {
Thread.sleep(ctx.getEdgeEventStorageSettings().getSleepIntervalBetweenBatches());
} catch (InterruptedException e) {
log.error("[{}] Error during sleep between batches", this.sessionId, e);
private ListenableFuture<Void> sendDownlinkMsgsPack(List<DownlinkMsg> downlinkMsgsPack) {
if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone()) {
String erroMsg = "[" + this.sessionId + "] Previous send downdlink future was not properly completed, stopping it now";
log.error(erroMsg);
sessionState.getSendDownlinkMsgsFuture().setException(new RuntimeException(erroMsg));
}
sessionState.setSendDownlinkMsgsFuture(SettableFuture.create());
sessionState.getPendingMsgsMap().clear();
downlinkMsgsPack.forEach(msg -> sessionState.getPendingMsgsMap().put(msg.getDownlinkMsgId(), msg));
scheduleDownlinkMsgsPackSend(true);
return sessionState.getSendDownlinkMsgsFuture();
}
private void scheduleDownlinkMsgsPackSend(boolean firstRun) {
Runnable sendDownlinkMsgsTask = () -> {
try {
if (isConnected() && sessionState.getPendingMsgsMap().values().size() > 0) {
if (!firstRun) {
log.warn("[{}] Failed to deliver the batch: {}", this.sessionId, sessionState.getPendingMsgsMap().values());
}
log.trace("[{}] [{}] downlink msg(s) are going to be send.", this.sessionId, sessionState.getPendingMsgsMap().values().size());
List<DownlinkMsg> copy = new ArrayList<>(sessionState.getPendingMsgsMap().values());
for (DownlinkMsg downlinkMsg : copy) {
sendDownlinkMsg(ResponseMsg.newBuilder()
.setDownlinkMsg(downlinkMsg)
.build());
}
scheduleDownlinkMsgsPackSend(false);
} else {
sessionState.getSendDownlinkMsgsFuture().set(null);
}
} while (isConnected() && (!success || pendingMsgsMap.values().size() > 0));
return success;
} finally {
downlinkMsgsPackLock.unlock();
} catch (Exception e) {
sessionState.getSendDownlinkMsgsFuture().setException(e);
}
};
if (firstRun) {
sendDownlinkExecutorService.submit(sendDownlinkMsgsTask);
} else {
sessionState.setScheduledSendDownlinkTask(
sendDownlinkExecutorService.schedule(
sendDownlinkMsgsTask,
ctx.getEdgeEventStorageSettings().getSleepIntervalBetweenBatches(),
TimeUnit.MILLISECONDS)
);
}
}
private DownlinkMsg convertToDownlinkMsg(EdgeEvent edgeEvent) {
@ -434,14 +496,14 @@ public final class EdgeGrpcSession implements Closeable {
} else {
return 0L;
}
}, ctx.getDbCallbackExecutor());
}, ctx.getGrpcCallbackExecutorService());
}
private void updateQueueStartTs(Long newStartTs) {
private ListenableFuture<List<Void>> updateQueueStartTs(Long newStartTs) {
log.trace("[{}] updating QueueStartTs [{}][{}]", this.sessionId, edge.getId(), newStartTs);
newStartTs = ++newStartTs; // increments ts by 1 - next edge event search starts from current offset + 1
List<AttributeKvEntry> attributes = Collections.singletonList(new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, newStartTs), System.currentTimeMillis()));
ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), DataConstants.SERVER_SCOPE, attributes);
return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), DataConstants.SERVER_SCOPE, attributes);
}
private DownlinkMsg processEntityMessage(EdgeEvent edgeEvent, EdgeEventActionType action) {

32
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSessionState.java

@ -0,0 +1,32 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edge.rpc;
import com.google.common.util.concurrent.SettableFuture;
import lombok.Data;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.ScheduledFuture;
@Data
public class EdgeSessionState {
private final Map<Integer, DownlinkMsg> pendingMsgsMap = new LinkedHashMap<>();
private SettableFuture<Void> sendDownlinkMsgsFuture;
private ScheduledFuture<?> scheduledSendDownlinkTask;
}

74
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java

@ -0,0 +1,74 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edge.rpc;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.service.edge.EdgeContextComponent;
import org.thingsboard.server.service.edge.rpc.fetch.AdminSettingsEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.AssetsEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.CustomerEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.CustomerUsersEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.DashboardsEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.DeviceProfilesEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.EdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.RuleChainsEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.SystemWidgetsBundlesEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.TenantAdminUsersEdgeEventFetcher;
import org.thingsboard.server.service.edge.rpc.fetch.TenantWidgetsBundlesEdgeEventFetcher;
import java.util.LinkedList;
import java.util.List;
import java.util.NoSuchElementException;
public class EdgeSyncCursor {
List<EdgeEventFetcher> fetchers = new LinkedList<>();
int currentIdx = 0;
public EdgeSyncCursor(EdgeContextComponent ctx, Edge edge) {
fetchers.add(new SystemWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService()));
fetchers.add(new TenantWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService()));
fetchers.add(new DeviceProfilesEdgeEventFetcher(ctx.getDeviceProfileService()));
fetchers.add(new RuleChainsEdgeEventFetcher(ctx.getRuleChainService()));
fetchers.add(new TenantAdminUsersEdgeEventFetcher(ctx.getUserService()));
if (edge.getCustomerId() != null && !EntityId.NULL_UUID.equals(edge.getCustomerId().getId())) {
fetchers.add(new CustomerEdgeEventFetcher());
fetchers.add(new CustomerUsersEdgeEventFetcher(ctx.getUserService(), edge.getCustomerId()));
}
fetchers.add(new AdminSettingsEdgeEventFetcher(ctx.getAdminSettingsService()));
fetchers.add(new AssetsEdgeEventFetcher(ctx.getAssetService()));
fetchers.add(new DashboardsEdgeEventFetcher(ctx.getDashboardService()));
}
public boolean hasNext() {
return fetchers.size() > currentIdx;
}
public EdgeEventFetcher getNext() {
if (!hasNext()) {
throw new NoSuchElementException();
}
EdgeEventFetcher edgeEventFetcher = fetchers.get(currentIdx);
currentIdx++;
return edgeEventFetcher;
}
public int getCurrentIdx() {
return currentIdx;
}
}

10
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java

@ -17,6 +17,7 @@ package org.thingsboard.server.service.edge.rpc.fetch;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -47,10 +48,17 @@ import java.util.regex.Pattern;
@AllArgsConstructor
@Slf4j
public class AdminSettingsEdgeEventFetcher extends BasePageableEdgeEventFetcher {
public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher {
private static final ObjectMapper mapper = new ObjectMapper();
private final AdminSettingsService adminSettingsService;
@Override
public PageLink getPageLink(int pageSize) {
return null;
}
@Override
public PageData<EdgeEvent> fetchEdgeEvents(TenantId tenantId, Edge edge, PageLink pageLink) throws Exception {
List<EdgeEvent> result = new ArrayList<>();

24
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AssetsEdgeEventFetcher.java

@ -28,26 +28,20 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.service.edge.rpc.EdgeEventUtils;
import java.util.ArrayList;
import java.util.List;
@AllArgsConstructor
@Slf4j
public class AssetsEdgeEventFetcher extends BasePageableEdgeEventFetcher {
public class AssetsEdgeEventFetcher extends BasePageableEdgeEventFetcher<Asset> {
private final AssetService assetService;
@Override
public PageData<EdgeEvent> fetchEdgeEvents(TenantId tenantId, Edge edge, PageLink pageLink) {
log.trace("[{}] start fetching edge events [{}]", tenantId, edge.getId());
PageData<Asset> pageData = assetService.findAssetsByTenantIdAndEdgeId(tenantId, edge.getId(), pageLink);
List<EdgeEvent> result = new ArrayList<>();
if (!pageData.getData().isEmpty()) {
for (Asset asset : pageData.getData()) {
result.add(EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ASSET,
EdgeEventActionType.ADDED, asset.getId(), null));
}
}
return new PageData<>(result, pageData.getTotalPages(), pageData.getTotalElements(), pageData.hasNext());
PageData<Asset> fetchPageData(TenantId tenantId, Edge edge, PageLink pageLink) {
return assetService.findAssetsByTenantIdAndEdgeId(tenantId, edge.getId(), pageLink);
}
@Override
EdgeEvent constructEdgeEvent(TenantId tenantId, Edge edge, Asset asset) {
return EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ASSET,
EdgeEventActionType.ADDED, asset.getId(), null);
}
}

40
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/BasePageableEdgeEventFetcher.java

@ -16,14 +16,50 @@
package org.thingsboard.server.service.edge.rpc.fetch;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.BaseData;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EventId;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.HasUUID;
import org.thingsboard.server.common.data.id.IdBased;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UUIDBased;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.service.edge.rpc.EdgeEventUtils;
public abstract class BasePageableEdgeEventFetcher implements EdgeEventFetcher {
import java.util.ArrayList;
import java.util.List;
protected static final ObjectMapper mapper = new ObjectMapper();
@Slf4j
public abstract class BasePageableEdgeEventFetcher<T> implements EdgeEventFetcher {
@Override
public PageLink getPageLink(int pageSize) {
return new PageLink(pageSize);
}
@Override
public PageData<EdgeEvent> fetchEdgeEvents(TenantId tenantId, Edge edge, PageLink pageLink) {
log.trace("[{}] start fetching edge events [{}]", tenantId, edge.getId());
PageData<T> pageData = fetchPageData(tenantId, edge, pageLink);
List<EdgeEvent> result = new ArrayList<>();
if (!pageData.getData().isEmpty()) {
for (T entity : pageData.getData()) {
result.add(constructEdgeEvent(tenantId, edge, entity));
}
}
return new PageData<>(result, pageData.getTotalPages(), pageData.getTotalElements(), pageData.hasNext());
}
abstract PageData<T> fetchPageData(TenantId tenantId, Edge edge, PageLink pageLink);
abstract EdgeEvent constructEdgeEvent(TenantId tenantId, Edge edge, T entity);
}

24
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/BaseUsersEdgeEventFetcher.java

@ -28,27 +28,21 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.service.edge.rpc.EdgeEventUtils;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@AllArgsConstructor
public abstract class BaseUsersEdgeEventFetcher extends BasePageableEdgeEventFetcher {
public abstract class BaseUsersEdgeEventFetcher extends BasePageableEdgeEventFetcher<User> {
protected final UserService userService;
@Override
public PageData<EdgeEvent> fetchEdgeEvents(TenantId tenantId, Edge edge, PageLink pageLink) {
log.trace("[{}] start fetching edge events [{}]", tenantId, edge.getId());
PageData<User> pageData = findUsers(tenantId, pageLink);
List<EdgeEvent> result = new ArrayList<>();
if (!pageData.getData().isEmpty()) {
for (User user : pageData.getData()) {
result.add(EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.USER,
EdgeEventActionType.ADDED, user.getId(), null));
}
}
return new PageData<>(result, pageData.getTotalPages(), pageData.getTotalElements(), pageData.hasNext());
PageData<User> fetchPageData(TenantId tenantId, Edge edge, PageLink pageLink) {
return findUsers(tenantId, pageLink);
}
@Override
EdgeEvent constructEdgeEvent(TenantId tenantId, Edge edge, User user) {
return EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.USER,
EdgeEventActionType.ADDED, user.getId(), null);
}
protected abstract PageData<User> findUsers(TenantId tenantId, PageLink pageLink);

24
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/BaseWidgetsBundlesEdgeEventFetcher.java

@ -28,27 +28,21 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.service.edge.rpc.EdgeEventUtils;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@AllArgsConstructor
public abstract class BaseWidgetsBundlesEdgeEventFetcher extends BasePageableEdgeEventFetcher {
public abstract class BaseWidgetsBundlesEdgeEventFetcher extends BasePageableEdgeEventFetcher<WidgetsBundle> {
protected final WidgetsBundleService widgetsBundleService;
@Override
public PageData<EdgeEvent> fetchEdgeEvents(TenantId tenantId, Edge edge, PageLink pageLink) {
log.trace("[{}] start fetching edge events [{}]", tenantId, edge.getId());
PageData<WidgetsBundle> pageData = findWidgetsBundles(tenantId, pageLink);
List<EdgeEvent> result = new ArrayList<>();
if (!pageData.getData().isEmpty()) {
for (WidgetsBundle widgetsBundle : pageData.getData()) {
result.add(EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.WIDGETS_BUNDLE,
EdgeEventActionType.ADDED, widgetsBundle.getId(), null));
}
}
return new PageData<>(result, pageData.getTotalPages(), pageData.getTotalElements(), pageData.hasNext());
PageData<WidgetsBundle> fetchPageData(TenantId tenantId, Edge edge, PageLink pageLink) {
return findWidgetsBundles(tenantId, pageLink);
}
@Override
EdgeEvent constructEdgeEvent(TenantId tenantId, Edge edge, WidgetsBundle widgetsBundle) {
return EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.WIDGETS_BUNDLE,
EdgeEventActionType.ADDED, widgetsBundle.getId(), null);
}
protected abstract PageData<WidgetsBundle> findWidgetsBundles(TenantId tenantId, PageLink pageLink);

49
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/CustomerEdgeEventFetcher.java

@ -0,0 +1,49 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edge.rpc.fetch;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.service.edge.rpc.EdgeEventUtils;
import java.util.ArrayList;
import java.util.List;
@AllArgsConstructor
@Slf4j
public class CustomerEdgeEventFetcher implements EdgeEventFetcher {
@Override
public PageLink getPageLink(int pageSize) {
return null;
}
@Override
public PageData<EdgeEvent> fetchEdgeEvents(TenantId tenantId, Edge edge, PageLink pageLink) {
List<EdgeEvent> result = new ArrayList<>();
result.add(EdgeEventUtils.constructEdgeEvent(edge.getTenantId(), edge.getId(),
EdgeEventType.CUSTOMER, EdgeEventActionType.ADDED, edge.getCustomerId(), null));
// @voba - returns PageData object to be in sync with other fetchers
return new PageData<>(result, 1, result.size(), false);
}
}

3
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/CustomerUsersEdgeEventFetcher.java

@ -16,6 +16,8 @@
package org.thingsboard.server.service.edge.rpc.fetch;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
@ -35,4 +37,5 @@ public class CustomerUsersEdgeEventFetcher extends BaseUsersEdgeEventFetcher {
protected PageData<User> findUsers(TenantId tenantId, PageLink pageLink) {
return userService.findCustomerUsers(tenantId, customerId, pageLink);
}
}

24
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/DashboardsEdgeEventFetcher.java

@ -28,26 +28,20 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.service.edge.rpc.EdgeEventUtils;
import java.util.ArrayList;
import java.util.List;
@AllArgsConstructor
@Slf4j
public class DashboardsEdgeEventFetcher extends BasePageableEdgeEventFetcher {
public class DashboardsEdgeEventFetcher extends BasePageableEdgeEventFetcher<DashboardInfo> {
private final DashboardService dashboardService;
@Override
public PageData<EdgeEvent> fetchEdgeEvents(TenantId tenantId, Edge edge, PageLink pageLink) {
log.trace("[{}] start fetching edge events [{}]", tenantId, edge.getId());
PageData<DashboardInfo> pageData = dashboardService.findDashboardsByTenantIdAndEdgeId(tenantId, edge.getId(), pageLink);
List<EdgeEvent> result = new ArrayList<>();
if (!pageData.getData().isEmpty()) {
for (DashboardInfo dashboardInfo : pageData.getData()) {
result.add(EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.DASHBOARD,
EdgeEventActionType.ADDED, dashboardInfo.getId(), null));
}
}
return new PageData<>(result, pageData.getTotalPages(), pageData.getTotalElements(), pageData.hasNext());
PageData<DashboardInfo> fetchPageData(TenantId tenantId, Edge edge, PageLink pageLink) {
return dashboardService.findDashboardsByTenantIdAndEdgeId(tenantId, edge.getId(), pageLink);
}
@Override
EdgeEvent constructEdgeEvent(TenantId tenantId, Edge edge, DashboardInfo dashboardInfo) {
return EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.DASHBOARD,
EdgeEventActionType.ADDED, dashboardInfo.getId(), null);
}
}

24
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/DeviceProfilesEdgeEventFetcher.java

@ -28,26 +28,20 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.service.edge.rpc.EdgeEventUtils;
import java.util.ArrayList;
import java.util.List;
@AllArgsConstructor
@Slf4j
public class DeviceProfilesEdgeEventFetcher extends BasePageableEdgeEventFetcher {
public class DeviceProfilesEdgeEventFetcher extends BasePageableEdgeEventFetcher<DeviceProfile> {
private final DeviceProfileService deviceProfileService;
@Override
public PageData<EdgeEvent> fetchEdgeEvents(TenantId tenantId, Edge edge, PageLink pageLink) {
log.trace("[{}] start fetching edge events [{}]", tenantId, edge.getId());
PageData<DeviceProfile> pageData = deviceProfileService.findDeviceProfiles(tenantId, pageLink);
List<EdgeEvent> result = new ArrayList<>();
if (!pageData.getData().isEmpty()) {
for (DeviceProfile deviceProfile : pageData.getData()) {
result.add(EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.DEVICE_PROFILE,
EdgeEventActionType.ADDED, deviceProfile.getId(), null));
}
}
return new PageData<>(result, pageData.getTotalPages(), pageData.getTotalElements(), pageData.hasNext());
PageData<DeviceProfile> fetchPageData(TenantId tenantId, Edge edge, PageLink pageLink) {
return deviceProfileService.findDeviceProfiles(tenantId, pageLink);
}
@Override
EdgeEvent constructEdgeEvent(TenantId tenantId, Edge edge, DeviceProfile deviceProfile) {
return EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.DEVICE_PROFILE,
EdgeEventActionType.ADDED, deviceProfile.getId(), null);
}
}

24
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/RuleChainsEdgeEventFetcher.java

@ -28,26 +28,20 @@ import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.service.edge.rpc.EdgeEventUtils;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@AllArgsConstructor
public class RuleChainsEdgeEventFetcher extends BasePageableEdgeEventFetcher {
public class RuleChainsEdgeEventFetcher extends BasePageableEdgeEventFetcher<RuleChain> {
private final RuleChainService ruleChainService;
@Override
public PageData<EdgeEvent> fetchEdgeEvents(TenantId tenantId, Edge edge, PageLink pageLink) {
log.trace("[{}] start fetching edge events [{}]", tenantId, edge.getId());
PageData<RuleChain> pageData = ruleChainService.findRuleChainsByTenantIdAndEdgeId(tenantId, edge.getId(), pageLink);
List<EdgeEvent> result = new ArrayList<>();
if (!pageData.getData().isEmpty()) {
for (RuleChain ruleChain : pageData.getData()) {
result.add(EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.RULE_CHAIN,
EdgeEventActionType.ADDED, ruleChain.getId(), null));
}
}
return new PageData<>(result, pageData.getTotalPages(), pageData.getTotalElements(), pageData.hasNext());
PageData<RuleChain> fetchPageData(TenantId tenantId, Edge edge, PageLink pageLink) {
return ruleChainService.findRuleChainsByTenantIdAndEdgeId(tenantId, edge.getId(), pageLink);
}
@Override
EdgeEvent constructEdgeEvent(TenantId tenantId, Edge edge, RuleChain ruleChain) {
return EdgeEventUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.RULE_CHAIN,
EdgeEventActionType.ADDED, ruleChain.getId(), null);
}
}

4
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/TenantWidgetsBundlesEdgeEventFetcher.java

@ -16,6 +16,8 @@
package org.thingsboard.server.service.edge.rpc.fetch;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
@ -23,7 +25,7 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
@Slf4j
public class TenantWidgetsBundlesEdgeEventFetcher extends BaseWidgetsBundlesEdgeEventFetcher implements EdgeEventFetcher {
public class TenantWidgetsBundlesEdgeEventFetcher extends BaseWidgetsBundlesEdgeEventFetcher {
public TenantWidgetsBundlesEdgeEventFetcher(WidgetsBundleService widgetsBundleService) {
super(widgetsBundleService);

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java

@ -189,8 +189,8 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor {
private Device createDevice(TenantId tenantId, Edge edge, DeviceUpdateMsg deviceUpdateMsg, String deviceName) {
Device device;
deviceCreationLock.lock();
try {
deviceCreationLock.lock();
log.debug("[{}] Creating device entity [{}] from edge [{}]", tenantId, deviceUpdateMsg, edge.getName());
DeviceId deviceId = new DeviceId(new UUID(deviceUpdateMsg.getIdMSB(), deviceUpdateMsg.getIdLSB()));
device = deviceService.findDeviceById(tenantId, deviceId);

33
application/src/main/java/org/thingsboard/server/service/executors/GrpcCallbackExecutorService.java

@ -0,0 +1,33 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.executors;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.common.util.AbstractListeningExecutor;
@Component
public class GrpcCallbackExecutorService extends AbstractListeningExecutor {
@Value("${edges.grpc_callback_thread_pool_size}")
private int grpcCallbackExecutorThreadPoolSize;
@Override
protected int getThreadPollSize() {
return grpcCallbackExecutorThreadPoolSize;
}
}

6
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@ -228,14 +228,14 @@ public class InstallScripts {
}
}
public void loadDemoRuleChains(TenantId tenantId) throws Exception {
public void loadDemoRuleChains(TenantId tenantId) {
try {
createDefaultRuleChains(tenantId);
createDefaultRuleChain(tenantId, "Thermostat");
createDefaultEdgeRuleChains(tenantId);
} catch (Exception e) {
log.error("Unable to load dashboard from json", e);
throw new RuntimeException("Unable to load dashboard from json", e);
log.error("Unable to load rule chain from json", e);
throw new RuntimeException("Unable to load rule chain from json", e);
}
}

16
application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
@ -22,9 +23,22 @@ import org.thingsboard.server.dao.util.PsqlDao;
@Service
@PsqlDao
@Profile("install")
@Slf4j
public class PsqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService
implements EntityDatabaseSchemaService {
public static final String SCHEMA_ENTITIES_SQL = "schema-entities.sql";
public static final String SCHEMA_ENTITIES_IDX_SQL = "schema-entities-idx.sql";
public static final String SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL = "schema-entities-idx-psql-addon.sql";
public PsqlEntityDatabaseSchemaService() {
super("schema-entities.sql", "schema-entities-idx.sql");
super(SCHEMA_ENTITIES_SQL, SCHEMA_ENTITIES_IDX_SQL);
}
@Override
public void createDatabaseIndexes() throws Exception {
super.createDatabaseIndexes();
log.info("Installing SQL DataBase schema PostgreSQL specific indexes part: " + SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL);
executeQueryFromFile(SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL);
}
}

27
application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java

@ -19,7 +19,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import java.nio.charset.Charset;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -59,14 +59,8 @@ public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchema
@Override
public void createDatabaseSchema(boolean createIndexes) throws Exception {
log.info("Installing SQL DataBase schema part: " + schemaSql);
Path schemaFile = Paths.get(installScripts.getDataDir(), SQL_DIR, schemaSql);
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
String sql = new String(Files.readAllBytes(schemaFile), Charset.forName("UTF-8"));
conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to load initial thingsboard database schema
}
executeQueryFromFile(schemaSql);
if (createIndexes) {
this.createDatabaseIndexes();
@ -77,11 +71,15 @@ public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchema
public void createDatabaseIndexes() throws Exception {
if (schemaIdxSql != null) {
log.info("Installing SQL DataBase schema indexes part: " + schemaIdxSql);
Path schemaIdxFile = Paths.get(installScripts.getDataDir(), SQL_DIR, schemaIdxSql);
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
String sql = new String(Files.readAllBytes(schemaIdxFile), Charset.forName("UTF-8"));
conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to load initial thingsboard database schema
}
executeQueryFromFile(schemaIdxSql);
}
}
void executeQueryFromFile(String schemaIdxSql) throws SQLException, IOException {
Path schemaIdxFile = Paths.get(installScripts.getDataDir(), SQL_DIR, schemaIdxSql);
String sql = Files.readString(schemaIdxFile);
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to load initial thingsboard database schema
}
}
@ -91,7 +89,8 @@ public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchema
log.info("Successfully executed query: {}", query);
Thread.sleep(5000);
} catch (InterruptedException | SQLException e) {
log.info("Failed to execute query: {} due to: {}", query, e.getMessage());
log.error("Failed to execute query: {} due to: {}", query, e.getMessage());
throw new RuntimeException("Failed to execute query: " + query, e);
}
}

4
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@ -459,6 +459,10 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", "schema_update_ttl.sql");
loadSql(schemaUpdateFile, conn);
log.info("Edge TTL functions successfully loaded!");
log.info("Updating indexes and TTL procedure for event table...");
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", "schema_update_event.sql");
loadSql(schemaUpdateFile, conn);
log.info("Updating schema settings...");
conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3003000;");
log.info("Schema updated.");
} catch (Exception e) {

82
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.service.install.update;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.alarm.AlarmInfo;
import org.thingsboard.server.common.data.alarm.AlarmQuery;
import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
@ -41,16 +43,21 @@ import org.thingsboard.server.common.data.oauth2.deprecated.OAuth2ClientsParams;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.common.data.query.DynamicValue;
import org.thingsboard.server.common.data.query.FilterPredicateValue;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.alarm.AlarmDao;
import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.model.sql.DeviceProfileEntity;
import org.thingsboard.server.dao.oauth2.OAuth2Service;
import org.thingsboard.server.dao.oauth2.OAuth2Utils;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.sql.device.DeviceProfileRepository;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.service.install.InstallScripts;
@ -92,6 +99,9 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired
private AlarmDao alarmDao;
@Autowired
private DeviceProfileRepository deviceProfileRepository;
@Autowired
private OAuth2Service oAuth2Service;
@ -114,6 +124,7 @@ public class DefaultDataUpdateService implements DataUpdateService {
log.info("Updating data from version 3.2.2 to 3.3.0 ...");
tenantsDefaultEdgeRuleChainUpdater.updateEntities(null);
tenantsAlarmsCustomerUpdater.updateEntities(null);
deviceProfileEntityDynamicConditionsUpdater.updateEntities(null);
updateOAuth2Params();
break;
default:
@ -121,6 +132,45 @@ public class DefaultDataUpdateService implements DataUpdateService {
}
}
private final PaginatedUpdater<String, DeviceProfileEntity> deviceProfileEntityDynamicConditionsUpdater =
new PaginatedUpdater<>() {
@Override
protected String getName() {
return "Device Profile Entity Dynamic Conditions Updater";
}
@Override
protected PageData<DeviceProfileEntity> findEntities(String id, PageLink pageLink) {
return DaoUtil.pageToPageData(deviceProfileRepository.findAll(DaoUtil.toPageable(pageLink)));
}
@Override
protected void updateEntity(DeviceProfileEntity deviceProfile) {
if (deviceProfile.getProfileData().has("alarms") &&
!deviceProfile.getProfileData().get("alarms").isNull()) {
boolean isUpdated = false;
JsonNode array = deviceProfile.getProfileData().get("alarms");
for (JsonNode node : array) {
if (node.has("createRules")) {
JsonNode createRules = node.get("createRules");
for (AlarmSeverity severity : AlarmSeverity.values()) {
if (createRules.has(severity.name())) {
isUpdated = isUpdated || convertDeviceProfileAlarmRulesForVersion330(createRules.get(severity.name()).get("condition").get("spec"));
}
}
}
if (node.has("clearRule") && !node.get("clearRule").isNull()) {
isUpdated = isUpdated || convertDeviceProfileAlarmRulesForVersion330(node.get("clearRule").get("condition").get("spec"));
}
}
if (isUpdated) {
deviceProfileRepository.save(deviceProfile);
}
}
}
};
private final PaginatedUpdater<String, Tenant> tenantsDefaultRuleChainUpdater =
new PaginatedUpdater<>() {
@ -370,6 +420,33 @@ public class DefaultDataUpdateService implements DataUpdateService {
}
}
private boolean convertDeviceProfileAlarmRulesForVersion330(JsonNode spec) {
if (spec != null) {
if (spec.has("type") && spec.get("type").asText().equals("DURATION")) {
if (spec.has("value")) {
long value = spec.get("value").asLong();
var predicate = new FilterPredicateValue<>(
value, null, new DynamicValue<>(null, null, false)
);
((ObjectNode) spec).remove("value");
((ObjectNode) spec).putPOJO("predicate", predicate);
return true;
}
} else if (spec.has("type") && spec.get("type").asText().equals("REPEATING")) {
if (spec.has("count")) {
int count = spec.get("count").asInt();
var predicate = new FilterPredicateValue<>(
count, null, new DynamicValue<>(null, null, false)
);
((ObjectNode) spec).remove("count");
((ObjectNode) spec).putPOJO("predicate", predicate);
return true;
}
}
}
return false;
}
private void updateOAuth2Params() {
try {
OAuth2ClientsParams oauth2ClientsParams = oAuth2Service.findOAuth2Params();
@ -380,9 +457,8 @@ public class DefaultDataUpdateService implements DataUpdateService {
oAuth2Service.saveOAuth2Params(new OAuth2ClientsParams(false, Collections.emptyList()));
log.info("Successfully updated OAuth2 parameters!");
}
}
catch (Exception e) {
log.error("Failed to update OAuth2 parameters", e);
} catch (Exception e) {
log.error("Failed to update OAuth2 parameters", e);
}
}

2
application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java

@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
@Slf4j
public abstract class PaginatedUpdater<I, D extends SearchTextBased<? extends UUIDBased>> {
public abstract class PaginatedUpdater<I, D> {
private static final int DEFAULT_LIMIT = 100;
private int updated = 0;

52
application/src/main/java/org/thingsboard/server/service/lwm2m/LwM2MServerSecurityInfoRepository.java

@ -30,8 +30,12 @@ import java.math.BigInteger;
import java.security.AlgorithmParameters;
import java.security.GeneralSecurityException;
import java.security.KeyFactory;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.PrivateKey;
import java.security.PublicKey;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateEncodingException;
import java.security.cert.X509Certificate;
import java.security.spec.ECGenParameterSpec;
@ -62,49 +66,23 @@ public class LwM2MServerSecurityInfoRepository {
bsServ.setPort(serverConfig.getPort());
bsServ.setSecurityHost(serverConfig.getSecureHost());
bsServ.setSecurityPort(serverConfig.getSecurePort());
bsServ.setServerPublicKey(getPublicKey(serverConfig.getCertificateAlias(), this.serverConfig.getPublicX(), this.serverConfig.getPublicY()));
bsServ.setServerPublicKey(getPublicKey(serverConfig));
return bsServ;
}
private String getPublicKey(String alias, String publicServerX, String publicServerY) {
String publicKey = getServerPublicKeyX509(alias);
return publicKey != null ? publicKey : getRPKPublicKey(publicServerX, publicServerY);
}
private String getServerPublicKeyX509(String alias) {
try {
X509Certificate serverCertificate = (X509Certificate) serverConfig.getKeyStoreValue().getCertificate(alias);
return Hex.encodeHexString(serverCertificate.getEncoded());
} catch (CertificateEncodingException | KeyStoreException e) {
e.printStackTrace();
}
return null;
}
private String getRPKPublicKey(String publicServerX, String publicServerY) {
private String getPublicKey(LwM2MSecureServerConfig config) {
try {
/** Get Elliptic Curve Parameter spec for secp256r1 */
AlgorithmParameters algoParameters = AlgorithmParameters.getInstance("EC");
algoParameters.init(new ECGenParameterSpec("secp256r1"));
ECParameterSpec parameterSpec = algoParameters.getParameterSpec(ECParameterSpec.class);
if (publicServerX != null && !publicServerX.isEmpty() && publicServerY != null && !publicServerY.isEmpty()) {
/** Get point values */
byte[] publicX = Hex.decodeHex(publicServerX.toCharArray());
byte[] publicY = Hex.decodeHex(publicServerY.toCharArray());
/** Create key specs */
KeySpec publicKeySpec = new ECPublicKeySpec(new ECPoint(new BigInteger(publicX), new BigInteger(publicY)),
parameterSpec);
/** Get keys */
PublicKey publicKey = KeyFactory.getInstance("EC").generatePublic(publicKeySpec);
if (publicKey != null && publicKey.getEncoded().length > 0) {
return Hex.encodeHexString(publicKey.getEncoded());
}
KeyStore keyStore = serverConfig.getKeyStoreValue();
if (keyStore != null) {
X509Certificate serverCertificate = (X509Certificate) serverConfig.getKeyStoreValue().getCertificate(config.getCertificateAlias());
return Hex.encodeHexString(serverCertificate.getPublicKey().getEncoded());
}
} catch (GeneralSecurityException | IllegalArgumentException e) {
log.error("[{}] Failed generate Server RPK for profile", e.getMessage());
throw new RuntimeException(e);
} catch (Exception e) {
log.trace("Failed to fetch public key from key store!", e);
}
return null;
return "";
}
}

72
application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java

@ -24,11 +24,14 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.MessageSource;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.NestedRuntimeException;
import org.springframework.core.io.InputStreamSource;
import org.springframework.mail.javamail.JavaMailSender;
import org.springframework.mail.javamail.JavaMailSenderImpl;
import org.springframework.mail.javamail.MimeMessageHelper;
import org.springframework.stereotype.Service;
import org.springframework.ui.freemarker.FreeMarkerTemplateUtils;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.TbEmail;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.ApiFeature;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
@ -45,8 +48,8 @@ import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import javax.annotation.PostConstruct;
import javax.mail.MessagingException;
import javax.mail.internet.MimeMessage;
import java.io.ByteArrayInputStream;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
@ -71,6 +74,9 @@ public class DefaultMailService implements MailService {
@Autowired
private TbApiUsageStateService apiUsageStateService;
@Autowired
private MailExecutorService mailExecutorService;
private JavaMailSenderImpl mailSender;
private String mailFrom;
@ -219,6 +225,17 @@ public class DefaultMailService implements MailService {
sendMail(mailSender, mailFrom, email, subject, message);
}
@Override
public void sendResetPasswordEmailAsync(String passwordResetLink, String email) {
mailExecutorService.execute(() -> {
try {
this.sendResetPasswordEmail(passwordResetLink, email);
} catch (ThingsboardException e) {
log.error("Error occurred: {} ", e.getMessage());
}
});
}
@Override
public void sendPasswordWasResetEmail(String loginLink, String email) throws ThingsboardException {
@ -234,22 +251,47 @@ public class DefaultMailService implements MailService {
}
@Override
public void send(TenantId tenantId, CustomerId customerId, String from, String to, String cc, String bcc, String subject, String body) throws MessagingException {
public void send(TenantId tenantId, CustomerId customerId, TbEmail tbEmail) throws ThingsboardException {
sendMail(tenantId, customerId, tbEmail, this.mailSender);
}
@Override
public void send(TenantId tenantId, CustomerId customerId, TbEmail tbEmail, JavaMailSender javaMailSender) throws ThingsboardException {
sendMail(tenantId, customerId, tbEmail, javaMailSender);
}
private void sendMail(TenantId tenantId, CustomerId customerId, TbEmail tbEmail, JavaMailSender javaMailSender) throws ThingsboardException {
if (apiUsageStateService.getApiUsageState(tenantId).isEmailSendEnabled()) {
MimeMessage mailMsg = mailSender.createMimeMessage();
MimeMessageHelper helper = new MimeMessageHelper(mailMsg, "UTF-8");
helper.setFrom(StringUtils.isBlank(from) ? mailFrom : from);
helper.setTo(to.split("\\s*,\\s*"));
if (!StringUtils.isBlank(cc)) {
helper.setCc(cc.split("\\s*,\\s*"));
try {
MimeMessage mailMsg = javaMailSender.createMimeMessage();
boolean multipart = (tbEmail.getImages() != null && !tbEmail.getImages().isEmpty());
MimeMessageHelper helper = new MimeMessageHelper(mailMsg, multipart, "UTF-8");
helper.setFrom(StringUtils.isBlank(tbEmail.getFrom()) ? mailFrom : tbEmail.getFrom());
helper.setTo(tbEmail.getTo().split("\\s*,\\s*"));
if (!StringUtils.isBlank(tbEmail.getCc())) {
helper.setCc(tbEmail.getCc().split("\\s*,\\s*"));
}
if (!StringUtils.isBlank(tbEmail.getBcc())) {
helper.setBcc(tbEmail.getBcc().split("\\s*,\\s*"));
}
helper.setSubject(tbEmail.getSubject());
helper.setText(tbEmail.getBody(), tbEmail.isHtml());
if (multipart) {
for (String imgId : tbEmail.getImages().keySet()) {
String imgValue = tbEmail.getImages().get(imgId);
String value = imgValue.replaceFirst("^data:image/[^;]*;base64,?", "");
byte[] bytes = javax.xml.bind.DatatypeConverter.parseBase64Binary(value);
String contentType = helper.getFileTypeMap().getContentType(imgId);
InputStreamSource iss = () -> new ByteArrayInputStream(bytes);
helper.addInline(imgId, iss, contentType);
}
}
javaMailSender.send(helper.getMimeMessage());
apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.EMAIL_EXEC_COUNT, 1);
} catch (Exception e) {
throw handleException(e);
}
if (!StringUtils.isBlank(bcc)) {
helper.setBcc(bcc.split("\\s*,\\s*"));
}
helper.setSubject(subject);
helper.setText(body);
mailSender.send(helper.getMimeMessage());
apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.EMAIL_EXEC_COUNT, 1);
} else {
throw new RuntimeException("Email sending is disabled due to API limits!");
}

8
application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java

@ -273,15 +273,19 @@ public class DefaultOtaPackageStateService implements OtaPackageStateService {
telemetryService.saveAndNotify(tenantId, deviceId, Collections.singletonList(status), new FutureCallback<>() {
@Override
public void onSuccess(@Nullable Void tmp) {
log.trace("[{}] Success save telemetry with target firmware for device!", deviceId);
log.trace("[{}] Success save telemetry with target {} for device!", deviceId, otaPackage);
updateAttributes(device, otaPackage, ts, tenantId, deviceId, otaPackageType);
}
@Override
public void onFailure(Throwable t) {
log.error("[{}] Failed to save telemetry with target firmware for device!", deviceId, t);
log.error("[{}] Failed to save telemetry with target {} for device!", deviceId, otaPackage, t);
updateAttributes(device, otaPackage, ts, tenantId, deviceId, otaPackageType);
}
});
}
private void updateAttributes(Device device, OtaPackageInfo otaPackage, long ts, TenantId tenantId, DeviceId deviceId, OtaPackageType otaPackageType) {
List<AttributeKvEntry> attributes = new ArrayList<>();
attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, TITLE), otaPackage.getTitle())));
attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, VERSION), otaPackage.getVersion())));

256
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -32,6 +32,7 @@ import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.gen.transport.TransportProtos;
@ -64,12 +65,14 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@Service
@ -79,12 +82,15 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
public static final String SUCCESSFUL_STATUS = "successful";
public static final String FAILED_STATUS = "failed";
public static final String THREAD_TOPIC_SEPARATOR = " | ";
@Value("${queue.rule-engine.poll-interval}")
private long pollDuration;
@Value("${queue.rule-engine.pack-processing-timeout}")
private long packProcessingTimeout;
@Value("${queue.rule-engine.stats.enabled:true}")
private boolean statsEnabled;
@Value("${queue.rule-engine.prometheus-stats.enabled:false}")
boolean prometheusStatsEnabled;
private final StatsFactory statsFactory;
private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory;
@ -96,7 +102,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
private final ConcurrentMap<String, TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>>> consumers = new ConcurrentHashMap<>();
private final ConcurrentMap<String, TbRuleEngineQueueConfiguration> consumerConfigurations = new ConcurrentHashMap<>();
private final ConcurrentMap<String, TbRuleEngineConsumerStats> consumerStats = new ConcurrentHashMap<>();
private ExecutorService submitExecutor;
private final ConcurrentMap<String, TbTopicWithConsumerPerPartition> topicsConsumerPerPartition = new ConcurrentHashMap<>();
final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-service-submit-executor"));
final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition-executor"));
public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory,
TbRuleEngineSubmitStrategyFactory submitStrategyFactory,
@ -125,18 +133,19 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer");
for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) {
consumerConfigurations.putIfAbsent(configuration.getName(), configuration);
consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration));
consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName(), statsFactory));
if (!configuration.isConsumerPerPartition()) {
consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration));
} else {
topicsConsumerPerPartition.computeIfAbsent(configuration.getName(), TbTopicWithConsumerPerPartition::new);
}
}
submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-service-submit-executor"));
}
@PreDestroy
public void stop() {
super.destroy();
if (submitExecutor != null) {
submitExecutor.shutdownNow();
}
submitExecutor.shutdownNow();
ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config));
}
@ -145,96 +154,185 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
if (event.getServiceType().equals(getServiceType())) {
ServiceQueue serviceQueue = event.getServiceQueueKey().getServiceQueue();
log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), event.getPartitions());
consumers.get(serviceQueue.getQueue()).subscribe(event.getPartitions());
if (!consumerConfigurations.get(serviceQueue.getQueue()).isConsumerPerPartition()) {
consumers.get(serviceQueue.getQueue()).subscribe(event.getPartitions());
} else {
log.info("[{}] Subscribing consumer per partition: {}", serviceQueue.getQueue(), event.getPartitions());
subscribeConsumerPerPartition(serviceQueue.getQueue(), event.getPartitions());
}
}
}
void subscribeConsumerPerPartition(String queue, Set<TopicPartitionInfo> partitions) {
topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions);
scheduleTopicRepartition(queue);
}
private void scheduleTopicRepartition(String queue) {
repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS);
}
void repartitionTopicWithConsumerPerPartition(final String queueName) {
if (stopped) {
return;
}
TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueName);
Queue<Set<TopicPartitionInfo>> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue();
if (subscribeQueue.isEmpty()) {
return;
}
if (tbTopicWithConsumerPerPartition.getLock().tryLock()) {
try {
Set<TopicPartitionInfo> partitions = null;
while (!subscribeQueue.isEmpty()) {
partitions = subscribeQueue.poll();
}
if (partitions == null) {
return;
}
Set<TopicPartitionInfo> addedPartitions = new HashSet<>(partitions);
ConcurrentMap<TopicPartitionInfo, TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>>> consumers = tbTopicWithConsumerPerPartition.getConsumers();
addedPartitions.removeAll(consumers.keySet());
log.info("calculated addedPartitions {}", addedPartitions);
Set<TopicPartitionInfo> removedPartitions = new HashSet<>(consumers.keySet());
removedPartitions.removeAll(partitions);
log.info("calculated removedPartitions {}", removedPartitions);
removedPartitions.forEach((tpi) -> {
removeConsumerForTopicByTpi(queueName, consumers, tpi);
});
addedPartitions.forEach((tpi) -> {
log.info("[{}] Adding consumer for topic: {}", queueName, tpi);
TbRuleEngineQueueConfiguration configuration = consumerConfigurations.get(queueName);
TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration);
consumers.put(tpi, consumer);
launchConsumer(consumer, consumerConfigurations.get(queueName), consumerStats.get(queueName), "" + queueName + "-" + tpi.getPartition().orElse(-999999));
consumer.subscribe(Collections.singleton(tpi));
});
} finally {
tbTopicWithConsumerPerPartition.getLock().unlock();
}
} else {
scheduleTopicRepartition(queueName); //reschedule later
}
}
void removeConsumerForTopicByTpi(String queue, ConcurrentMap<TopicPartitionInfo, TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>>> consumers, TopicPartitionInfo tpi) {
log.info("[{}] Removing consumer for topic: {}", queue, tpi);
consumers.get(tpi).unsubscribe();
consumers.remove(tpi);
}
@Override
protected void launchMainConsumers() {
consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue)));
consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue), queue));
}
@Override
protected void stopMainConsumers() {
consumers.values().forEach(TbQueueConsumer::unsubscribe);
topicsConsumerPerPartition.values().forEach(tbTopicWithConsumerPerPartition -> tbTopicWithConsumerPerPartition.getConsumers().keySet()
.forEach((tpi) -> removeConsumerForTopicByTpi(tbTopicWithConsumerPerPartition.getTopic(), tbTopicWithConsumerPerPartition.getConsumers(), tpi)));
}
void launchConsumer(TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats, String threadSuffix) {
consumersExecutor.execute(() -> consumerLoop(consumer, configuration, stats, threadSuffix));
}
private void launchConsumer(TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats) {
consumersExecutor.execute(() -> {
Thread.currentThread().setName("" + Thread.currentThread().getName() + "-" + configuration.getName());
while (!stopped) {
try {
List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs = consumer.poll(pollDuration);
if (msgs.isEmpty()) {
continue;
void consumerLoop(TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats, String threadSuffix) {
updateCurrentThreadName(threadSuffix);
while (!stopped && !consumer.isStopped()) {
try {
List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs = consumer.poll(pollDuration);
if (msgs.isEmpty()) {
continue;
}
final TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(configuration);
final TbRuleEngineProcessingStrategy ackStrategy = getAckStrategy(configuration);
submitStrategy.init(msgs);
while (!stopped) {
TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(configuration.getName(), submitStrategy);
submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> submitMessage(configuration, stats, ctx, id, msg)));
final boolean timeout = !ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS);
TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, ctx);
if (timeout) {
printFirstOrAll(configuration, ctx, ctx.getPendingMap(), "Timeout");
}
TbRuleEngineSubmitStrategy submitStrategy = submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy());
TbRuleEngineProcessingStrategy ackStrategy = processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy());
submitStrategy.init(msgs);
while (!stopped) {
TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(configuration.getName(), submitStrategy);
submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> {
log.trace("[{}] Creating callback for message: {}", id, msg.getValue());
ToRuleEngineMsg toRuleEngineMsg = msg.getValue();
TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB()));
TbMsgCallback callback = statsEnabled ?
new TbMsgPackCallback(id, tenantId, ctx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) :
new TbMsgPackCallback(id, tenantId, ctx);
try {
if (!toRuleEngineMsg.getTbMsg().isEmpty()) {
forwardToRuleEngineActor(configuration.getName(), tenantId, toRuleEngineMsg, callback);
} else {
callback.onSuccess();
}
} catch (Exception e) {
callback.onFailure(new RuleEngineException(e.getMessage()));
}
}));
boolean timeout = false;
if (!ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
timeout = true;
}
TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, ctx);
if (timeout) {
printFirstOrAll(configuration, ctx, ctx.getPendingMap(), "Timeout");
}
if (!ctx.getFailedMap().isEmpty()) {
printFirstOrAll(configuration, ctx, ctx.getFailedMap(), "Failed");
}
ctx.printProfilerStats();
TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result);
if (statsEnabled) {
stats.log(result, decision.isCommit());
}
ctx.cleanup();
if (decision.isCommit()) {
submitStrategy.stop();
break;
} else {
submitStrategy.update(decision.getReprocessMap());
}
if (!ctx.getFailedMap().isEmpty()) {
printFirstOrAll(configuration, ctx, ctx.getFailedMap(), "Failed");
}
consumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to process messages from queue.", e);
try {
Thread.sleep(pollDuration);
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new requests", e2);
}
ctx.printProfilerStats();
TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result);
if (statsEnabled) {
stats.log(result, decision.isCommit());
}
ctx.cleanup();
if (decision.isCommit()) {
submitStrategy.stop();
break;
} else {
submitStrategy.update(decision.getReprocessMap());
}
}
consumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to process messages from queue.", e);
try {
Thread.sleep(pollDuration);
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new requests", e2);
}
}
}
log.info("TB Rule Engine Consumer stopped.");
});
}
log.info("TB Rule Engine Consumer stopped.");
}
void updateCurrentThreadName(String threadSuffix) {
String name = Thread.currentThread().getName();
int spliteratorIndex = name.indexOf(THREAD_TOPIC_SEPARATOR);
if (spliteratorIndex > 0) {
name = name.substring(0, spliteratorIndex);
}
name = name + THREAD_TOPIC_SEPARATOR + threadSuffix;
Thread.currentThread().setName(name);
}
TbRuleEngineProcessingStrategy getAckStrategy(TbRuleEngineQueueConfiguration configuration) {
return processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy());
}
TbRuleEngineSubmitStrategy getSubmitStrategy(TbRuleEngineQueueConfiguration configuration) {
return submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy());
}
void submitMessage(TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats, TbMsgPackProcessingContext ctx, UUID id, TbProtoQueueMsg<ToRuleEngineMsg> msg) {
log.trace("[{}] Creating callback for topic {} message: {}", id, configuration.getName(), msg.getValue());
ToRuleEngineMsg toRuleEngineMsg = msg.getValue();
TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB()));
TbMsgCallback callback = prometheusStatsEnabled ?
new TbMsgPackCallback(id, tenantId, ctx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) :
new TbMsgPackCallback(id, tenantId, ctx);
try {
if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) {
forwardToRuleEngineActor(configuration.getName(), tenantId, toRuleEngineMsg, callback);
} else {
callback.onSuccess();
}
} catch (Exception e) {
callback.onFailure(new RuleEngineException(e.getMessage()));
}
}
private void printFirstOrAll(TbRuleEngineQueueConfiguration configuration, TbMsgPackProcessingContext ctx, Map<UUID, TbProtoQueueMsg<ToRuleEngineMsg>> map, String prefix) {

45
application/src/main/java/org/thingsboard/server/service/queue/TbTopicWithConsumerPerPartition.java

@ -0,0 +1,45 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import java.util.Collections;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantLock;
@RequiredArgsConstructor
@Data
public class TbTopicWithConsumerPerPartition {
private final String topic;
@Getter
private final ReentrantLock lock = new ReentrantLock(); //NonfairSync
private volatile Set<TopicPartitionInfo> partitions = Collections.emptySet();
private final ConcurrentMap<TopicPartitionInfo, TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>>> consumers = new ConcurrentHashMap<>();
private final Queue<Set<TopicPartitionInfo>> subscribeQueue = new ConcurrentLinkedQueue<>();
}

11
application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java

@ -86,7 +86,10 @@ public class DefaultTbResourceService implements TbResourceService {
} else {
throw new DataValidationException(String.format("Could not parse the XML of objectModel with name %s", resource.getSearchText()));
}
} catch (InvalidDDFFileException | IOException e) {
} catch (InvalidDDFFileException e) {
log.error("Failed to parse file {}", resource.getFileName(), e);
throw new DataValidationException("Failed to parse file " + resource.getFileName());
} catch (IOException e) {
throw new ThingsboardException(e, ThingsboardErrorCode.GENERAL);
}
if (resource.getResourceType().equals(ResourceType.LWM2M_MODEL) && toLwM2mObject(resource, true) == null) {
@ -194,8 +197,7 @@ public class DefaultTbResourceService implements TbResourceService {
if (isSave) {
LwM2mResourceObserve lwM2MResourceObserve = new LwM2mResourceObserve(k, v.name, false, false, false);
resources.add(lwM2MResourceObserve);
}
else if (v.operations.isReadable()) {
} else if (v.operations.isReadable()) {
LwM2mResourceObserve lwM2MResourceObserve = new LwM2mResourceObserve(k, v.name, false, false, false);
resources.add(lwM2MResourceObserve);
}
@ -204,8 +206,7 @@ public class DefaultTbResourceService implements TbResourceService {
instance.setResources(resources.toArray(LwM2mResourceObserve[]::new));
lwM2mObject.setInstances(new LwM2mInstance[]{instance});
return lwM2mObject;
}
else {
} else {
return null;
}
}

1
application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java

@ -157,6 +157,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
metaData.putValue("originServiceId", serviceId);
metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime()));
metaData.putValue("oneway", Boolean.toString(msg.isOneway()));
metaData.putValue(DataConstants.PERSISTENT, Boolean.toString(msg.isPersisted()));
Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId());
if (device != null) {

2
application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java

@ -100,7 +100,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi
@Override
public void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest src, Consumer<RuleEngineDeviceRpcResponse> consumer) {
ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), src.getTenantId(), src.getDeviceId(),
src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody()));
src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody()), src.isPersisted());
forwardRpcRequestToDeviceActor(request, response -> {
if (src.isRestApiCall()) {
sendRpcResponseToTbCore(src.getOriginServiceId(), response);

77
application/src/main/java/org/thingsboard/server/service/rpc/TbRpcService.java

@ -0,0 +1,77 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.rpc;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.RpcId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rpc.Rpc;
import org.thingsboard.server.common.data.rpc.RpcStatus;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.rpc.RpcService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.queue.TbClusterService;
@TbCoreComponent
@Service
@RequiredArgsConstructor
@Slf4j
public class TbRpcService {
private final RpcService rpcService;
private final TbClusterService tbClusterService;
public Rpc save(TenantId tenantId, Rpc rpc) {
Rpc saved = rpcService.save(rpc);
pushRpcMsgToRuleEngine(tenantId, saved);
return saved;
}
public void save(TenantId tenantId, RpcId rpcId, RpcStatus newStatus, JsonNode response) {
Rpc foundRpc = rpcService.findById(tenantId, rpcId);
if (foundRpc != null) {
foundRpc.setStatus(newStatus);
if (response != null) {
foundRpc.setResponse(response);
}
Rpc saved = rpcService.save(foundRpc);
pushRpcMsgToRuleEngine(tenantId, saved);
} else {
log.warn("[{}] Failed to update RPC status because RPC was already deleted", rpcId);
}
}
private void pushRpcMsgToRuleEngine(TenantId tenantId, Rpc rpc) {
TbMsg msg = TbMsg.newMsg("RPC_" + rpc.getStatus().name(), rpc.getDeviceId(), TbMsgMetaData.EMPTY, JacksonUtil.toString(rpc));
tbClusterService.pushMsgToRuleEngine(tenantId, rpc.getId(), msg, null);
}
public Rpc findRpcById(TenantId tenantId, RpcId rpcId) {
return rpcService.findById(tenantId, rpcId);
}
public PageData<Rpc> findAllByDeviceIdAndStatus(TenantId tenantId, DeviceId deviceId, RpcStatus rpcStatus, PageLink pageLink) {
return rpcService.findAllByDeviceIdAndStatus(tenantId, deviceId, rpcStatus, pageLink);
}
}

25
application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java

@ -47,11 +47,13 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.id.RpcId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.rpc.Rpc;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.controller.HttpValidationCallback;
@ -65,6 +67,7 @@ import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rpc.RpcService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
@ -137,6 +140,9 @@ public class AccessValidator {
@Autowired
protected OtaPackageService otaPackageService;
@Autowired
protected RpcService rpcService;
private ExecutorService executor;
@PostConstruct
@ -235,6 +241,9 @@ public class AccessValidator {
case OTA_PACKAGE:
validateOtaPackage(currentUser, operation, entityId, callback);
return;
case RPC:
validateRpc(currentUser, operation, entityId, callback);
return;
default:
//TODO: add support of other entities
throw new IllegalStateException("Not Implemented!");
@ -261,6 +270,22 @@ public class AccessValidator {
}
}
private void validateRpc(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
ListenableFuture<Rpc> rpcFurure = rpcService.findRpcByIdAsync(currentUser.getTenantId(), new RpcId(entityId.getId()));
Futures.addCallback(rpcFurure, getCallback(callback, rpc -> {
if (rpc == null) {
return ValidationResult.entityNotFound("Rpc with requested id wasn't found!");
} else {
try {
accessControlService.checkPermission(currentUser, Resource.RPC, operation, entityId, rpc);
} catch (ThingsboardException e) {
return ValidationResult.accessDenied(e.getMessage());
}
return ValidationResult.ok(rpc);
}
}), executor);
}
private void validateDeviceProfile(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
if (currentUser.isSystemAdmin()) {
callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION));

6
application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java

@ -204,11 +204,7 @@ public abstract class AbstractOAuth2ClientMapper {
}
private Optional<DashboardId> getDashboardId(TenantId tenantId, String dashboardName) {
PageLink searchTextLink = new PageLink(1, 0, dashboardName);
PageData<DashboardInfo> dashboardsPage = dashboardService.findDashboardsByTenantId(tenantId, searchTextLink);
return dashboardsPage.getData().stream()
.findAny()
.map(IdBased::getId);
return Optional.ofNullable(dashboardService.findFirstDashboardInfoByTenantIdAndName(tenantId, dashboardName)).map(IdBased::getId);
}
private Optional<DashboardId> getDashboardId(TenantId tenantId, CustomerId customerId, String dashboardName) {

19
application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java

@ -41,6 +41,7 @@ public class CustomerUserPermissions extends AbstractPermissions {
put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker);
put(Resource.WIDGET_TYPE, widgetsPermissionChecker);
put(Resource.EDGE, customerEntityPermissionChecker);
put(Resource.RPC, rpcPermissionChecker);
}
private static final PermissionChecker customerEntityPermissionChecker =
@ -138,4 +139,22 @@ public class CustomerUserPermissions extends AbstractPermissions {
}
};
private static final PermissionChecker rpcPermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ) {
@Override
@SuppressWarnings("unchecked")
public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) {
if (!super.hasPermission(user, operation, entityId, entity)) {
return false;
}
if (entity.getTenantId() == null || entity.getTenantId().isNullUid()) {
return true;
}
if (!user.getTenantId().equals(entity.getTenantId())) {
return false;
}
return true;
}
};
}

2
application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java

@ -19,6 +19,6 @@ public enum Operation {
ALL, CREATE, READ, WRITE, DELETE, ASSIGN_TO_CUSTOMER, UNASSIGN_FROM_CUSTOMER, RPC_CALL,
READ_CREDENTIALS, WRITE_CREDENTIALS, READ_ATTRIBUTES, WRITE_ATTRIBUTES, READ_TELEMETRY, WRITE_TELEMETRY, CLAIM_DEVICES,
ASSIGN_TO_TENANT, ASSIGN_TO_EDGE, UNASSIGN_FROM_EDGE
ASSIGN_TO_TENANT
}

3
application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java

@ -39,7 +39,8 @@ public enum Resource {
API_USAGE_STATE(EntityType.API_USAGE_STATE),
TB_RESOURCE(EntityType.TB_RESOURCE),
OTA_PACKAGE(EntityType.OTA_PACKAGE),
EDGE(EntityType.EDGE);
EDGE(EntityType.EDGE),
RPC(EntityType.RPC);
private final EntityType entityType;

1
application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java

@ -44,6 +44,7 @@ public class TenantAdminPermissions extends AbstractPermissions {
put(Resource.TB_RESOURCE, tbResourcePermissionChecker);
put(Resource.OTA_PACKAGE, tenantEntityPermissionChecker);
put(Resource.EDGE, tenantEntityPermissionChecker);
put(Resource.RPC, tenantEntityPermissionChecker);
}
public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() {

390
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@ -22,6 +22,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@ -60,8 +61,8 @@ import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
import org.thingsboard.server.utils.EventDeduplicationExecutor;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
@ -70,12 +71,16 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Queue;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@ -128,13 +133,13 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
@Getter
private int initFetchPackSize;
private ListeningScheduledExecutorService queueExecutor;
private ListeningScheduledExecutorService scheduledExecutor;
private ExecutorService deviceStateExecutor;
private final ConcurrentMap<TopicPartitionInfo, Set<DeviceId>> partitionedDevices = new ConcurrentHashMap<>();
private final ConcurrentMap<DeviceId, DeviceStateData> deviceStates = new ConcurrentHashMap<>();
private final ConcurrentMap<DeviceId, Long> deviceLastReportedActivity = new ConcurrentHashMap<>();
final ConcurrentMap<DeviceId, DeviceStateData> deviceStates = new ConcurrentHashMap<>();
private final ConcurrentMap<DeviceId, Long> deviceLastSavedActivity = new ConcurrentHashMap<>();
private volatile EventDeduplicationExecutor<Set<TopicPartitionInfo>> deduplicationExecutor;
final Queue<Set<TopicPartitionInfo>> subscribeQueue = new ConcurrentLinkedQueue<>();
public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService,
AttributesService attributesService, TimeseriesService tsService,
@ -154,16 +159,20 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
@PostConstruct
public void init() {
deviceStateExecutor = Executors.newFixedThreadPool(
Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("device-state"));
// Should be always single threaded due to absence of locks.
queueExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-state")));
queueExecutor.scheduleAtFixedRate(this::updateState, new Random().nextInt(defaultStateCheckIntervalInSec), defaultStateCheckIntervalInSec, TimeUnit.SECONDS);
deduplicationExecutor = new EventDeduplicationExecutor<>(DefaultDeviceStateService.class.getSimpleName(), queueExecutor, this::initStateFromDB);
scheduledExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-state-scheduled")));
scheduledExecutor.scheduleAtFixedRate(this::updateInactivityStateIfExpired, new Random().nextInt(defaultStateCheckIntervalInSec), defaultStateCheckIntervalInSec, TimeUnit.SECONDS);
}
@PreDestroy
public void stop() {
if (queueExecutor != null) {
queueExecutor.shutdownNow();
if (deviceStateExecutor != null) {
deviceStateExecutor.shutdownNow();
}
if (scheduledExecutor != null) {
scheduledExecutor.shutdownNow();
}
}
@ -184,62 +193,60 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
@Override
public void onDeviceConnect(DeviceId deviceId) {
log.trace("on Device Connect [{}]", deviceId.getId());
DeviceStateData stateData = getOrFetchDeviceStateData(deviceId);
if (stateData != null) {
long ts = System.currentTimeMillis();
stateData.getState().setLastConnectTime(ts);
pushRuleEngineMessage(stateData, CONNECT_EVENT);
save(deviceId, LAST_CONNECT_TIME, ts);
}
long ts = System.currentTimeMillis();
stateData.getState().setLastConnectTime(ts);
save(deviceId, LAST_CONNECT_TIME, ts);
pushRuleEngineMessage(stateData, CONNECT_EVENT);
checkAndUpdateState(deviceId, stateData);
}
@Override
public void onDeviceActivity(DeviceId deviceId, long lastReportedActivity) {
deviceLastReportedActivity.put(deviceId, lastReportedActivity);
log.trace("on Device Activity [{}], lastReportedActivity [{}]", deviceId.getId(), lastReportedActivity);
long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L);
if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) {
DeviceStateData stateData = getOrFetchDeviceStateData(deviceId);
if (stateData != null) {
save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity);
deviceLastSavedActivity.put(deviceId, lastReportedActivity);
DeviceState state = stateData.getState();
state.setLastActivityTime(lastReportedActivity);
if (!state.isActive()) {
state.setActive(true);
save(deviceId, ACTIVITY_STATE, state.isActive());
pushRuleEngineMessage(stateData, ACTIVITY_EVENT);
}
final DeviceStateData stateData = getOrFetchDeviceStateData(deviceId);
updateActivityState(deviceId, stateData, lastReportedActivity);
}
}
void updateActivityState(DeviceId deviceId, DeviceStateData stateData, long lastReportedActivity) {
log.trace("updateActivityState - fetched state {} for device {}, lastReportedActivity {}", stateData, deviceId, lastReportedActivity);
if (stateData != null) {
save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity);
deviceLastSavedActivity.put(deviceId, lastReportedActivity);
DeviceState state = stateData.getState();
state.setLastActivityTime(lastReportedActivity);
if (!state.isActive()) {
state.setActive(true);
save(deviceId, ACTIVITY_STATE, true);
pushRuleEngineMessage(stateData, ACTIVITY_EVENT);
}
} else {
log.warn("updateActivityState - fetched state IN NULL for device {}, lastReportedActivity {}", deviceId, lastReportedActivity);
}
}
@Override
public void onDeviceDisconnect(DeviceId deviceId) {
DeviceStateData stateData = getOrFetchDeviceStateData(deviceId);
if (stateData != null) {
long ts = System.currentTimeMillis();
stateData.getState().setLastDisconnectTime(ts);
pushRuleEngineMessage(stateData, DISCONNECT_EVENT);
save(deviceId, LAST_DISCONNECT_TIME, ts);
}
long ts = System.currentTimeMillis();
stateData.getState().setLastDisconnectTime(ts);
save(deviceId, LAST_DISCONNECT_TIME, ts);
pushRuleEngineMessage(stateData, DISCONNECT_EVENT);
}
@Override
public void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) {
if (inactivityTimeout == 0L) {
if (inactivityTimeout <= 0L) {
return;
}
DeviceStateData stateData = deviceStates.get(deviceId);
if (stateData != null) {
long ts = System.currentTimeMillis();
DeviceState state = stateData.getState();
state.setInactivityTimeout(inactivityTimeout);
boolean oldActive = state.isActive();
state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout());
if (!oldActive && state.isActive() || oldActive && !state.isActive()) {
save(deviceId, ACTIVITY_STATE, state.isActive());
}
}
log.trace("on Device Activity Timeout Update device id {} inactivityTimeout {}", deviceId, inactivityTimeout);
DeviceStateData stateData = getOrFetchDeviceStateData(deviceId);
stateData.getState().setInactivityTimeout(inactivityTimeout);
checkAndUpdateState(deviceId, stateData);
}
@Override
@ -273,7 +280,7 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
log.warn("Failed to register device to the state service", t);
callback.onFailure(t);
}
}, MoreExecutors.directExecutor());
}, deviceStateExecutor);
} else if (proto.getUpdated()) {
DeviceStateData stateData = getOrFetchDeviceStateData(device.getId());
if (stateData != null) {
@ -294,25 +301,55 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
}
}
/**
* DiscoveryService will call this event from the single thread (one-by-one).
* Events order is guaranteed by DiscoveryService.
* The only concurrency is expected from the [main] thread on Application started.
* Async implementation. Locks is not allowed by design.
* Any locks or delays in this module will affect DiscoveryService and entire system
*/
@Override
protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) {
deduplicationExecutor.submit(partitionChangeEvent.getPartitions());
log.debug("onTbApplicationEvent ServiceType is TB_CORE, processing queue {}", partitionChangeEvent);
subscribeQueue.add(partitionChangeEvent.getPartitions());
scheduledExecutor.submit(this::pollInitStateFromDB);
}
}
void pollInitStateFromDB() {
final Set<TopicPartitionInfo> partitions = getLatestPartitionsFromQueue();
if (partitions == null) {
log.info("Device state service. Nothing to do. partitions is null");
return;
}
initStateFromDB(partitions);
}
// TODO: move to utils
Set<TopicPartitionInfo> getLatestPartitionsFromQueue() {
log.debug("getLatestPartitionsFromQueue, queue size {}", subscribeQueue.size());
Set<TopicPartitionInfo> partitions = null;
while (!subscribeQueue.isEmpty()) {
partitions = subscribeQueue.poll();
log.debug("polled from the queue partitions {}", partitions);
}
log.debug("getLatestPartitionsFromQueue, partitions {}", partitions);
return partitions;
}
private void initStateFromDB(Set<TopicPartitionInfo> pendingPartitions) {
private void initStateFromDB(Set<TopicPartitionInfo> partitions) {
try {
log.info("CURRENT PARTITIONS: {}", partitionedDevices.keySet());
log.info("NEW PARTITIONS: {}", pendingPartitions);
log.info("NEW PARTITIONS: {}", partitions);
Set<TopicPartitionInfo> addedPartitions = new HashSet<>(pendingPartitions);
Set<TopicPartitionInfo> addedPartitions = new HashSet<>(partitions);
addedPartitions.removeAll(partitionedDevices.keySet());
log.info("ADDED PARTITIONS: {}", addedPartitions);
Set<TopicPartitionInfo> removedPartitions = new HashSet<>(partitionedDevices.keySet());
removedPartitions.removeAll(pendingPartitions);
removedPartitions.removeAll(partitions);
log.info("REMOVED PARTITIONS: {}", removedPartitions);
@ -321,45 +358,14 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
Set<DeviceId> devices = partitionedDevices.remove(partition);
devices.forEach(deviceId -> {
deviceStates.remove(deviceId);
deviceLastReportedActivity.remove(deviceId);
deviceLastSavedActivity.remove(deviceId);
});
});
addedPartitions.forEach(tpi -> partitionedDevices.computeIfAbsent(tpi, key -> ConcurrentHashMap.newKeySet()));
//TODO 3.0: replace this dummy search with new functionality to search by partitions using SQL capabilities.
// Adding only devices that are in new partitions
List<Tenant> tenants = tenantService.findTenants(new PageLink(Integer.MAX_VALUE)).getData();
for (Tenant tenant : tenants) {
PageLink pageLink = new PageLink(initFetchPackSize);
while (pageLink != null) {
List<ListenableFuture<Void>> fetchFutures = new ArrayList<>();
PageData<Device> page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink);
pageLink = page.hasNext() ? pageLink.nextPageLink() : null;
for (Device device : page.getData()) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), device.getId());
if (addedPartitions.contains(tpi)) {
ListenableFuture<Void> future = Futures.transform(fetchDeviceState(device), new Function<DeviceStateData, Void>() {
@Nullable
@Override
public Void apply(@Nullable DeviceStateData state) {
if (state != null) {
addDeviceUsingState(tpi, state);
}
return null;
}
}, MoreExecutors.directExecutor());
fetchFutures.add(future);
}
}
try {
Futures.successfulAsList(fetchFutures).get();
} catch (InterruptedException | ExecutionException e) {
log.warn("Failed to init device state service from DB", e);
}
}
}
initPartitions(addedPartitions);
log.info("Managing following partitions:");
partitionedDevices.forEach((tpi, devices) -> {
log.info("[{}]: {} devices", tpi.getFullTopicName(), devices.size());
@ -369,49 +375,151 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
}
}
//TODO 3.0: replace this dummy search with new functionality to search by partitions using SQL capabilities.
//Adding only entities that are in new partitions
boolean initPartitions(Set<TopicPartitionInfo> addedPartitions) {
if (addedPartitions.isEmpty()) {
return false;
}
List<Tenant> tenants = tenantService.findTenants(new PageLink(Integer.MAX_VALUE)).getData();
for (Tenant tenant : tenants) {
log.debug("Finding devices for tenant [{}]", tenant.getName());
final PageLink pageLink = new PageLink(initFetchPackSize);
scheduledExecutor.submit(() -> processPageAndSubmitNextPage(addedPartitions, tenant, pageLink, scheduledExecutor));
}
return true;
}
private void processPageAndSubmitNextPage(final Set<TopicPartitionInfo> addedPartitions, final Tenant tenant, final PageLink pageLink, final ExecutorService executor) {
log.trace("[{}] Process page {} from {}", tenant, pageLink.getPage(), pageLink.getPageSize());
List<ListenableFuture<Void>> fetchFutures = new ArrayList<>();
PageData<Device> page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink);
for (Device device : page.getData()) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), device.getId());
if (addedPartitions.contains(tpi)) {
log.debug("[{}][{}] Device belong to current partition. tpi [{}]. Fetching state from DB", device.getName(), device.getId(), tpi);
ListenableFuture<Void> future = Futures.transform(fetchDeviceState(device), new Function<DeviceStateData, Void>() {
@Nullable
@Override
public Void apply(@Nullable DeviceStateData state) {
if (state != null) {
addDeviceUsingState(tpi, state);
checkAndUpdateState(device.getId(), state);
} else {
log.warn("{}][{}] Fetched null state from DB", device.getName(), device.getId());
}
return null;
}
}, deviceStateExecutor);
fetchFutures.add(future);
} else {
log.debug("[{}][{}] Device doesn't belong to current partition. tpi [{}]", device.getName(), device.getId(), tpi);
}
}
Futures.addCallback(Futures.successfulAsList(fetchFutures), new FutureCallback<List<Void>>() {
@Override
public void onSuccess(List<Void> result) {
log.trace("[{}] Success init device state from DB for batch size {}", tenant.getId(), result.size());
}
@Override
public void onFailure(Throwable t) {
log.warn("[" + tenant.getId() + "] Failed to init device state service from DB", t);
log.warn("[{}] Failed to init device state service from DB", tenant.getId(), t);
}
}, deviceStateExecutor);
final PageLink nextPageLink = page.hasNext() ? pageLink.nextPageLink() : null;
if (nextPageLink != null) {
log.trace("[{}] Submit next page {} from {}", tenant, nextPageLink.getPage(), nextPageLink.getPageSize());
executor.submit(() -> processPageAndSubmitNextPage(addedPartitions, tenant, nextPageLink, executor));
}
}
void checkAndUpdateState(@Nonnull DeviceId deviceId, @Nonnull DeviceStateData state) {
if (state.getState().isActive()) {
updateInactivityStateIfExpired(System.currentTimeMillis(), deviceId, state);
} else {
//trying to fix activity state
if (isActive(System.currentTimeMillis(), state.getState())) {
updateActivityState(deviceId, state, state.getState().getLastActivityTime());
}
}
}
private void addDeviceUsingState(TopicPartitionInfo tpi, DeviceStateData state) {
partitionedDevices.computeIfAbsent(tpi, id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId());
deviceStates.put(state.getDeviceId(), state);
Set<DeviceId> deviceIds = partitionedDevices.get(tpi);
if (deviceIds != null) {
deviceIds.add(state.getDeviceId());
deviceStates.put(state.getDeviceId(), state);
} else {
log.warn("Device belongs to external partition {}" + tpi.getFullTopicName());
new RuntimeException("Device belongs to external partition " + tpi.getFullTopicName() + "!");
}
}
private void updateState() {
long ts = System.currentTimeMillis();
Set<DeviceId> deviceIds = new HashSet<>(deviceStates.keySet());
void updateInactivityStateIfExpired() {
final long ts = System.currentTimeMillis();
log.debug("Calculating state updates for {} devices", deviceStates.size());
Set<DeviceId> deviceIds = new HashSet<>(deviceStates.keySet());
for (DeviceId deviceId : deviceIds) {
DeviceStateData stateData = getOrFetchDeviceStateData(deviceId);
if (stateData != null) {
DeviceState state = stateData.getState();
state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout());
if (!state.isActive() && (state.getLastInactivityAlarmTime() == 0L || state.getLastInactivityAlarmTime() < state.getLastActivityTime()) && stateData.getDeviceCreationTime() + state.getInactivityTimeout() < ts) {
state.setLastInactivityAlarmTime(ts);
pushRuleEngineMessage(stateData, INACTIVITY_EVENT);
save(deviceId, INACTIVITY_ALARM_TIME, ts);
save(deviceId, ACTIVITY_STATE, state.isActive());
}
} else {
log.debug("[{}] Device that belongs to other server is detected and removed.", deviceId);
deviceStates.remove(deviceId);
deviceLastReportedActivity.remove(deviceId);
deviceLastSavedActivity.remove(deviceId);
updateInactivityStateIfExpired(ts, deviceId);
}
}
void updateInactivityStateIfExpired(long ts, DeviceId deviceId) {
DeviceStateData stateData = getOrFetchDeviceStateData(deviceId);
updateInactivityStateIfExpired(ts, deviceId, stateData);
}
void updateInactivityStateIfExpired(long ts, DeviceId deviceId, DeviceStateData stateData) {
log.trace("Processing state {} for device {}", stateData, deviceId);
if (stateData != null) {
DeviceState state = stateData.getState();
if (!isActive(ts, state) && (state.getLastInactivityAlarmTime() == 0L || state.getLastInactivityAlarmTime() < state.getLastActivityTime()) && stateData.getDeviceCreationTime() + state.getInactivityTimeout() < ts) {
state.setActive(false);
state.setLastInactivityAlarmTime(ts);
save(deviceId, INACTIVITY_ALARM_TIME, ts);
save(deviceId, ACTIVITY_STATE, false);
pushRuleEngineMessage(stateData, INACTIVITY_EVENT);
}
} else {
log.debug("[{}] Device that belongs to other server is detected and removed.", deviceId);
deviceStates.remove(deviceId);
deviceLastSavedActivity.remove(deviceId);
}
}
private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) {
boolean isActive(long ts, DeviceState state) {
return ts < state.getLastActivityTime() + state.getInactivityTimeout();
}
@Nonnull
DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) {
DeviceStateData deviceStateData = deviceStates.get(deviceId);
if (deviceStateData == null) {
Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId);
if (device != null) {
try {
deviceStateData = fetchDeviceState(device).get();
deviceStates.putIfAbsent(deviceId, deviceStateData);
} catch (InterruptedException | ExecutionException e) {
log.debug("[{}] Failed to fetch device state!", deviceId, e);
}
}
if (deviceStateData != null) {
return deviceStateData;
}
return fetchDeviceStateData(deviceId);
}
DeviceStateData fetchDeviceStateData(final DeviceId deviceId) {
final Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId);
if (device == null) {
log.warn("[{}] Failed to fetch device by Id!", deviceId);
throw new RuntimeException("Failed to fetch device by Id " + deviceId);
}
try {
DeviceStateData deviceStateData = fetchDeviceState(device).get();
deviceStates.putIfAbsent(deviceId, deviceStateData);
return deviceStateData;
} catch (InterruptedException | ExecutionException e) {
log.warn("[{}] Failed to fetch device state!", deviceId, e);
throw new RuntimeException(e);
}
return deviceStateData;
}
private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, boolean added, boolean updated, boolean deleted) {
@ -429,7 +537,6 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) {
deviceStates.remove(deviceId);
deviceLastReportedActivity.remove(deviceId);
deviceLastSavedActivity.remove(deviceId);
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId);
Set<DeviceId> deviceIdSet = partitionedDevices.get(tpi);
@ -437,25 +544,45 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
}
private ListenableFuture<DeviceStateData> fetchDeviceState(Device device) {
ListenableFuture<DeviceStateData> future;
if (persistToTelemetry) {
ListenableFuture<List<TsKvEntry>> tsData = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), PERSISTENT_ATTRIBUTES);
return Futures.transform(tsData, extractDeviceStateData(device), MoreExecutors.directExecutor());
future = Futures.transform(tsData, extractDeviceStateData(device), deviceStateExecutor);
} else {
ListenableFuture<List<AttributeKvEntry>> attrData = attributesService.find(TenantId.SYS_TENANT_ID, device.getId(), DataConstants.SERVER_SCOPE, PERSISTENT_ATTRIBUTES);
return Futures.transform(attrData, extractDeviceStateData(device), MoreExecutors.directExecutor());
future = Futures.transform(attrData, extractDeviceStateData(device), deviceStateExecutor);
}
return transformInactivityTimeout(future);
}
private ListenableFuture<DeviceStateData> transformInactivityTimeout(ListenableFuture<DeviceStateData> future) {
return Futures.transformAsync(future, deviceStateData -> {
if (!persistToTelemetry || deviceStateData.getState().getInactivityTimeout() != TimeUnit.SECONDS.toMillis(defaultInactivityTimeoutInSec)) {
return future; //fail fast
}
var attributesFuture = attributesService.find(TenantId.SYS_TENANT_ID, deviceStateData.getDeviceId(), SERVER_SCOPE, INACTIVITY_TIMEOUT);
return Futures.transform(attributesFuture, attributes -> {
attributes.flatMap(KvEntry::getLongValue).ifPresent((inactivityTimeout) -> {
if (inactivityTimeout > 0) {
deviceStateData.getState().setInactivityTimeout(inactivityTimeout);
}
});
return deviceStateData;
}, deviceStateExecutor);
}, deviceStateExecutor);
}
private <T extends KvEntry> Function<List<T>, DeviceStateData> extractDeviceStateData(Device device) {
return new Function<>() {
@Nullable
return new Function<List<T>, DeviceStateData>() {
@Nonnull
@Override
public DeviceStateData apply(@Nullable List<T> data) {
try {
long lastActivityTime = getEntryValue(data, LAST_ACTIVITY_TIME, 0L);
long inactivityAlarmTime = getEntryValue(data, INACTIVITY_ALARM_TIME, 0L);
long inactivityTimeout = getEntryValue(data, INACTIVITY_TIMEOUT, TimeUnit.SECONDS.toMillis(defaultInactivityTimeoutInSec));
boolean active = System.currentTimeMillis() < lastActivityTime + inactivityTimeout;
//Actual active state by wall-clock will updated outside this method. This method is only for fetch persistent state
final boolean active = getEntryValue(data, ACTIVITY_STATE, false);
DeviceState deviceState = DeviceState.builder()
.active(active)
.lastConnectTime(getEntryValue(data, LAST_CONNECT_TIME, 0L))
@ -467,13 +594,15 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
TbMsgMetaData md = new TbMsgMetaData();
md.putValue("deviceName", device.getName());
md.putValue("deviceType", device.getType());
return DeviceStateData.builder()
DeviceStateData deviceStateData = DeviceStateData.builder()
.customerId(device.getCustomerId())
.tenantId(device.getTenantId())
.deviceId(device.getId())
.deviceCreationTime(device.getCreatedTime())
.metaData(md)
.state(deviceState).build();
log.debug("[{}] Fetched device state from the DB {}", device.getId(), deviceStateData);
return deviceStateData;
} catch (Exception e) {
log.warn("[{}] Failed to fetch device state data", device.getId(), e);
throw new RuntimeException(e);
@ -493,6 +622,17 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
return defaultValue;
}
private boolean getEntryValue(List<? extends KvEntry> kvEntries, String attributeName, boolean defaultValue) {
if (kvEntries != null) {
for (KvEntry entry : kvEntries) {
if (entry != null && !StringUtils.isEmpty(entry.getKey()) && entry.getKey().equals(attributeName)) {
return entry.getBooleanValue().orElse(defaultValue);
}
}
}
return defaultValue;
}
private void pushRuleEngineMessage(DeviceStateData stateData, String msgType) {
DeviceState state = stateData.getState();
try {
@ -505,7 +645,7 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
data = JacksonUtil.toString(state);
}
TbMsgMetaData md = stateData.getMetaData().copy();
if(!persistToTelemetry){
if (!persistToTelemetry) {
md.putValue(DataConstants.SCOPE, SERVER_SCOPE);
}
TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getCustomerId(), md, TbMsgDataType.JSON, data);

50
application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java

@ -22,6 +22,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.ByteString;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
@ -40,14 +41,16 @@ import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials;
import org.thingsboard.server.common.data.device.credentials.ProvisionDeviceCredentialsData;
import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration;
import org.thingsboard.server.common.data.device.data.PowerMode;
import org.thingsboard.server.common.data.device.profile.ProvisionDeviceProfileCredentials;
import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.ota.OtaPackageUtil;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.ota.OtaPackageUtil;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.relation.EntityRelation;
@ -108,6 +111,7 @@ import java.util.stream.Collectors;
@Slf4j
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class DefaultTransportApiService implements TransportApiService {
private static final ObjectMapper mapper = new ObjectMapper();
@ -129,28 +133,6 @@ public class DefaultTransportApiService implements TransportApiService {
private final ConcurrentMap<String, ReentrantLock> deviceCreationLocks = new ConcurrentHashMap<>();
public DefaultTransportApiService(TbDeviceProfileCache deviceProfileCache,
TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, DeviceService deviceService,
RelationService relationService, DeviceCredentialsService deviceCredentialsService,
DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService,
TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService,
DeviceProvisionService deviceProvisionService, TbResourceService resourceService, OtaPackageService otaPackageService, OtaPackageDataCache otaPackageDataCache) {
this.deviceProfileCache = deviceProfileCache;
this.tenantProfileCache = tenantProfileCache;
this.apiUsageStateService = apiUsageStateService;
this.deviceService = deviceService;
this.relationService = relationService;
this.deviceCredentialsService = deviceCredentialsService;
this.deviceStateService = deviceStateService;
this.dbCallbackExecutorService = dbCallbackExecutorService;
this.tbClusterService = tbClusterService;
this.dataDecodingEncodingService = dataDecodingEncodingService;
this.deviceProvisionService = deviceProvisionService;
this.resourceService = resourceService;
this.otaPackageService = otaPackageService;
this.otaPackageDataCache = otaPackageDataCache;
}
@Override
public ListenableFuture<TbProtoQueueMsg<TransportApiResponseMsg>> handle(TbProtoQueueMsg<TransportApiRequestMsg> tbProtoQueueMsg) {
TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue();
@ -479,7 +461,17 @@ public class DefaultTransportApiService implements TransportApiService {
}
private DeviceInfoProto getDeviceInfoProto(Device device) throws JsonProcessingException {
return DeviceInfoProto.newBuilder()
PowerMode powerMode = null;
Long edrxCycle = null;
switch (device.getDeviceData().getTransportConfiguration().getType()) {
case LWM2M:
Lwm2mDeviceTransportConfiguration transportConfiguration = (Lwm2mDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration();
powerMode = transportConfiguration.getPowerMode();
edrxCycle = transportConfiguration.getEdrxCycle();
break;
}
DeviceInfoProto.Builder builder = DeviceInfoProto.newBuilder()
.setTenantIdMSB(device.getTenantId().getId().getMostSignificantBits())
.setTenantIdLSB(device.getTenantId().getId().getLeastSignificantBits())
.setCustomerIdMSB(Optional.ofNullable(device.getCustomerId()).map(customerId -> customerId.getId().getMostSignificantBits()).orElse(0L))
@ -490,8 +482,12 @@ public class DefaultTransportApiService implements TransportApiService {
.setDeviceType(device.getType())
.setDeviceProfileIdMSB(device.getDeviceProfileId().getId().getMostSignificantBits())
.setDeviceProfileIdLSB(device.getDeviceProfileId().getId().getLeastSignificantBits())
.setAdditionalInfo(mapper.writeValueAsString(device.getAdditionalInfo()))
.build();
.setAdditionalInfo(mapper.writeValueAsString(device.getAdditionalInfo()));
if (powerMode != null) {
builder.setPowerMode(powerMode.name());
builder.setEdrxCycle(edrxCycle);
}
return builder.build();
}
private ListenableFuture<TransportApiResponseMsg> getEmptyTransportApiResponseFuture() {

44
application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java

@ -15,8 +15,13 @@
*/
package org.thingsboard.server.service.ttl;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.queue.discovery.PartitionService;
import java.sql.Connection;
import java.sql.DriverManager;
@ -27,43 +32,12 @@ import java.sql.Statement;
@Slf4j
@RequiredArgsConstructor
public abstract class AbstractCleanUpService {
@Value("${spring.datasource.url}")
protected String dbUrl;
private final PartitionService partitionService;
@Value("${spring.datasource.username}")
protected String dbUserName;
@Value("${spring.datasource.password}")
protected String dbPassword;
protected long executeQuery(Connection conn, String query) throws SQLException {
try (Statement statement = conn.createStatement(); ResultSet resultSet = statement.executeQuery(query)) {
if (log.isDebugEnabled()) {
getWarnings(statement);
}
resultSet.next();
return resultSet.getLong(1);
}
}
protected void getWarnings(Statement statement) throws SQLException {
SQLWarning warnings = statement.getWarnings();
if (warnings != null) {
log.debug("{}", warnings.getMessage());
SQLWarning nextWarning = warnings.getNextWarning();
while (nextWarning != null) {
log.debug("{}", nextWarning.getMessage());
nextWarning = nextWarning.getNextWarning();
}
}
protected boolean isSystemTenantPartitionMine(){
return partitionService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition();
}
protected abstract void doCleanUp(Connection connection) throws SQLException;
protected Connection getConnection() throws SQLException {
return DriverManager.getConnection(dbUrl, dbUserName, dbPassword);
}
}

3
application/src/main/java/org/thingsboard/server/service/ttl/alarms/AlarmsCleanUpService.java → application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl.alarms;
package org.thingsboard.server.service.ttl;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -52,6 +52,7 @@ import java.util.concurrent.TimeUnit;
@Slf4j
@RequiredArgsConstructor
public class AlarmsCleanUpService {
@Value("${sql.ttl.alarms.removal_batch_size}")
private Integer removalBatchSize;

33
application/src/main/java/org/thingsboard/server/service/ttl/edge/EdgeEventsCleanUpService.java → application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java

@ -13,20 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl.edge;
package org.thingsboard.server.service.ttl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.service.ttl.AbstractCleanUpService;
import org.thingsboard.server.dao.edge.EdgeEventService;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@PsqlDao
@TbCoreComponent
@Slf4j
@Service
public class EdgeEventsCleanUpService extends AbstractCleanUpService {
@ -37,20 +34,18 @@ public class EdgeEventsCleanUpService extends AbstractCleanUpService {
@Value("${sql.ttl.edge_events.enabled}")
private boolean ttlTaskExecutionEnabled;
private final EdgeEventService edgeEventService;
public EdgeEventsCleanUpService(PartitionService partitionService, EdgeEventService edgeEventService) {
super(partitionService);
this.edgeEventService = edgeEventService;
}
@Scheduled(initialDelayString = "${sql.ttl.edge_events.execution_interval_ms}", fixedDelayString = "${sql.ttl.edge_events.execution_interval_ms}")
public void cleanUp() {
if (ttlTaskExecutionEnabled) {
try (Connection conn = getConnection()) {
doCleanUp(conn);
} catch (SQLException e) {
log.error("SQLException occurred during TTL task execution ", e);
}
if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) {
edgeEventService.cleanupEvents(ttl);
}
}
@Override
protected void doCleanUp(Connection connection) throws SQLException {
long totalEdgeEventsRemoved = executeQuery(connection, "call cleanup_edge_events_by_ttl(" + ttl + ", 0);");
log.info("Total edge events removed by TTL: [{}]", totalEdgeEventsRemoved);
}
}

37
application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java → application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java

@ -13,24 +13,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl.events;
package org.thingsboard.server.service.ttl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.event.EventService;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.ttl.AbstractCleanUpService;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@PsqlDao
@TbCoreComponent
@Slf4j
@Service
public class EventsCleanUpService extends AbstractCleanUpService {
public static final String RANDOM_DELAY_INTERVAL_MS_EXPRESSION =
"#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.events.execution_interval_ms})}";
@Value("${sql.ttl.events.events_ttl}")
private long ttl;
@ -40,20 +41,18 @@ public class EventsCleanUpService extends AbstractCleanUpService {
@Value("${sql.ttl.events.enabled}")
private boolean ttlTaskExecutionEnabled;
@Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}")
private final EventService eventService;
public EventsCleanUpService(PartitionService partitionService, EventService eventService) {
super(partitionService);
this.eventService = eventService;
}
@Scheduled(initialDelayString = RANDOM_DELAY_INTERVAL_MS_EXPRESSION, fixedDelayString = "${sql.ttl.events.execution_interval_ms}")
public void cleanUp() {
if (ttlTaskExecutionEnabled) {
try (Connection conn = getConnection()) {
doCleanUp(conn);
} catch (SQLException e) {
log.error("SQLException occurred during TTL task execution ", e);
}
if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) {
eventService.cleanupEvents(ttl, debugTtl);
}
}
@Override
protected void doCleanUp(Connection connection) throws SQLException {
long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);");
log.info("Total events removed by TTL: [{}]", totalEventsRemoved);
}
}

29
application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java → application/src/main/java/org/thingsboard/server/service/ttl/TimeseriesCleanUpService.java

@ -13,19 +13,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl.timeseries;
package org.thingsboard.server.service.ttl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.ttl.AbstractCleanUpService;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@TbCoreComponent
@Slf4j
public abstract class AbstractTimeseriesCleanUpService extends AbstractCleanUpService {
@Service
public class TimeseriesCleanUpService extends AbstractCleanUpService {
@Value("${sql.ttl.ts.ts_key_value_ttl}")
protected long systemTtl;
@ -33,14 +35,17 @@ public abstract class AbstractTimeseriesCleanUpService extends AbstractCleanUpSe
@Value("${sql.ttl.ts.enabled}")
private boolean ttlTaskExecutionEnabled;
private final TimeseriesService timeseriesService;
public TimeseriesCleanUpService(PartitionService partitionService, TimeseriesService timeseriesService) {
super(partitionService);
this.timeseriesService = timeseriesService;
}
@Scheduled(initialDelayString = "${sql.ttl.ts.execution_interval_ms}", fixedDelayString = "${sql.ttl.ts.execution_interval_ms}")
public void cleanUp() {
if (ttlTaskExecutionEnabled) {
try (Connection conn = getConnection()) {
doCleanUp(conn);
} catch (SQLException e) {
log.error("SQLException occurred during TTL task execution ", e);
}
if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) {
timeseriesService.cleanup(systemTtl);
}
}

83
application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java

@ -0,0 +1,83 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl.rpc;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.dao.rpc.RpcDao;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.dao.tenant.TenantDao;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import java.util.Date;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@TbCoreComponent
@Service
@Slf4j
@RequiredArgsConstructor
public class RpcCleanUpService {
@Value("${sql.ttl.rpc.enabled}")
private boolean ttlTaskExecutionEnabled;
private final TenantDao tenantDao;
private final PartitionService partitionService;
private final TbTenantProfileCache tenantProfileCache;
private final RpcDao rpcDao;
@Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.rpc.checking_interval})}", fixedDelayString = "${sql.ttl.rpc.checking_interval}")
public void cleanUp() {
if (ttlTaskExecutionEnabled) {
PageLink tenantsBatchRequest = new PageLink(10_000, 0);
PageData<TenantId> tenantsIds;
do {
tenantsIds = tenantDao.findTenantsIds(tenantsBatchRequest);
for (TenantId tenantId : tenantsIds.getData()) {
if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) {
continue;
}
Optional<DefaultTenantProfileConfiguration> tenantProfileConfiguration = tenantProfileCache.get(tenantId).getProfileConfiguration();
if (tenantProfileConfiguration.isEmpty() || tenantProfileConfiguration.get().getRpcTtlDays() == 0) {
continue;
}
long ttl = TimeUnit.DAYS.toMillis(tenantProfileConfiguration.get().getRpcTtlDays());
long expirationTime = System.currentTimeMillis() - ttl;
long totalRemoved = rpcDao.deleteOutdatedRpcByTenantId(tenantId, expirationTime);
if (totalRemoved > 0) {
log.info("Removed {} outdated rpc(s) for tenant {} older than {}", totalRemoved, tenantId, new Date(expirationTime));
}
}
tenantsBatchRequest = tenantsBatchRequest.nextPageLink();
} while (tenantsIds.hasNext());
}
}
}

44
application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java

@ -1,44 +0,0 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl.timeseries;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.util.SqlTsDao;
import java.sql.Connection;
import java.sql.SQLException;
@SqlTsDao
@PsqlDao
@Service
@Slf4j
public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService {
@Value("${sql.postgres.ts_key_value_partitioning}")
private String partitionType;
@Override
protected void doCleanUp(Connection connection) throws SQLException {
long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);");
log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved);
long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID + "'," + systemTtl + ", 0);");
log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved);
}
}

36
application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java

@ -1,36 +0,0 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl.timeseries;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.util.TimescaleDBTsDao;
import java.sql.Connection;
import java.sql.SQLException;
@TimescaleDBTsDao
@Service
@Slf4j
public class TimescaleTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService {
@Override
protected void doCleanUp(Connection connection) throws SQLException {
long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID + "'," + systemTtl + ", 0);");
log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved);
}
}

1
application/src/main/resources/logback.xml

@ -26,7 +26,6 @@
</appender>
<logger name="org.thingsboard.server" level="INFO" />
<logger name="org.thingsboard.server.transport.snmp" level="DEBUG" />
<!-- <logger name="org.thingsboard.server.service.queue" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.service.transport" level="TRACE" />-->

61
application/src/main/resources/thingsboard.yml

@ -266,7 +266,7 @@ sql:
ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds
events:
enabled: "${SQL_TTL_EVENTS_ENABLED:true}"
execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of milliseconds. The current value corresponds to one day
execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:2220000}" # Number of milliseconds (max random initial delay and fixed period). # 37minutes to avoid common interval spikes
events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds
debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week
edge_events:
@ -276,6 +276,9 @@ sql:
alarms:
checking_interval: "${SQL_ALARMS_TTL_CHECKING_INTERVAL:7200000}" # Number of milliseconds. The current value corresponds to two hours
removal_batch_size: "${SQL_ALARMS_TTL_REMOVAL_BATCH_SIZE:3000}" # To delete outdated alarms not all at once but in batches
rpc:
enabled: "${SQL_TTL_RPC_ENABLED:true}"
checking_interval: "${SQL_RPC_TTL_CHECKING_INTERVAL:7200000}" # Number of milliseconds. The current value corresponds to two hours
# Actor system parameters
actors:
@ -515,7 +518,7 @@ audit-log:
index_pattern: "${AUDIT_LOG_SINK_INDEX_PATTERN:@{TENANT}_AUDIT_LOG_@{DATE}}"
# Date format. Details of the pattern could be found here:
# https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html
date_format: "${AUDIT_LOG_SINK_DATE_FORMAT:YYYY.MM.DD}"
date_format: "${AUDIT_LOG_SINK_DATE_FORMAT:YYYY.MM.dd}"
scheme_name: "${AUDIT_LOG_SINK_SCHEME_NAME:http}" # http or https
host: "${AUDIT_LOG_SINK_HOST:localhost}"
port: "${AUDIT_LOG_SINK_PORT:9200}"
@ -592,6 +595,10 @@ transport:
ssl:
# Enable/disable SSL support
enabled: "${MQTT_SSL_ENABLED:false}"
# MQTT SSL bind address
bind_address: "${MQTT_SSL_BIND_ADDRESS:0.0.0.0}"
# MQTT SSL bind port
bind_port: "${MQTT_SSL_BIND_PORT:8883}"
# SSL protocol: See http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#SSLContext
protocol: "${MQTT_SSL_PROTOCOL:TLSv1.2}"
# Path to the key store that holds the SSL certificate
@ -618,8 +625,6 @@ transport:
bind_address: "${COAP_DTLS_BIND_ADDRESS:0.0.0.0}"
# CoAP DTLS bind port
bind_port: "${COAP_DTLS_BIND_PORT:5684}"
# Secure mode. Allowed values: NO_AUTH, X509
mode: "${COAP_DTLS_SECURE_MODE:NO_AUTH}"
# Path to the key store that holds the certificate
key_store: "${COAP_DTLS_KEY_STORE:coapserver.jks}"
# Password used to access the key store
@ -642,48 +647,40 @@ transport:
bind_address: "${LWM2M_BIND_ADDRESS:0.0.0.0}"
bind_port: "${LWM2M_BIND_PORT:5685}"
security:
bind_address: "${LWM2M_BIND_ADDRESS_SECURITY:0.0.0.0}"
bind_port: "${LWM2M_BIND_PORT_SECURITY:5686}"
# Only for RPK: Public & Private Key. If the keystore file is missing or not working
public_x: "${LWM2M_SERVER_PUBLIC_X:05064b9e6762dd8d8b8a52355d7b4d8b9a3d64e6d2ee277d76c248861353f358}"
public_y: "${LWM2M_SERVER_PUBLIC_Y:5eeb1838e4f9e37b31fa347aef5ce3431eb54e0a2506910c5e0298817445721b}"
private_encoded: "${LWM2M_SERVER_PRIVATE_ENCODED:308193020100301306072a8648ce3d020106082a8648ce3d030107047930770201010420dc774b309e547ceb48fee547e104ce201a9c48c449dc5414cd04e7f5cf05f67ba00a06082a8648ce3d030107a1440342000405064b9e6762dd8d8b8a52355d7b4d8b9a3d64e6d2ee277d76c248861353f3585eeb1838e4f9e37b31fa347aef5ce3431eb54e0a2506910c5e0298817445721b}"
bind_address: "${LWM2M_SECURITY_BIND_ADDRESS:0.0.0.0}"
bind_port: "${LWM2M_SECURITY_BIND_PORT:5686}"
# Only Certificate_x509:
alias: "${LWM2M_KEYSTORE_ALIAS_SERVER:server}"
key_alias: "${LWM2M_SERVER_KEY_ALIAS:server}"
key_password: "${LWM2M_SERVER_KEY_PASSWORD:server_ks_password}"
skip_validity_check_for_client_cert: "${TB_LWM2M_SERVER_SECURITY_SKIP_VALIDITY_CHECK_FOR_CLIENT_CERT:false}"
bootstrap:
enable: "${LWM2M_ENABLED_BS:true}"
id: "${LWM2M_SERVER_ID_BS:111}"
bind_address: "${LWM2M_BIND_ADDRESS_BS:0.0.0.0}"
bind_port: "${LWM2M_BIND_PORT_BS:5687}"
bind_address: "${LWM2M_BS_BIND_ADDRESS:0.0.0.0}"
bind_port: "${LWM2M_BS_BIND_PORT:5687}"
security:
bind_address: "${LWM2M_BIND_ADDRESS_SECURITY_BS:0.0.0.0}"
bind_port: "${LWM2M_BIND_PORT_SECURITY_BS:5688}"
# Only for RPK: Public & Private Key. If the keystore file is missing or not working
public_x: "${LWM2M_SERVER_PUBLIC_X_BS:5017c87a1c1768264656b3b355434b0def6edb8b9bf166a4762d9930cd730f91}"
public_y: "${LWM2M_SERVER_PUBLIC_Y_BS:3fc4e61bcd8901ec27c424114c3e887ed372497f0c2cf85839b8443e76988b34}"
private_encoded: "${LWM2M_SERVER_PRIVATE_ENCODED_BS:308193020100301306072a8648ce3d020106082a8648ce3d0301070479307702010104205ecafd90caa7be45c42e1f3f32571632b8409e6e6249d7124f4ba56fab3c8083a00a06082a8648ce3d030107a144034200045017c87a1c1768264656b3b355434b0def6edb8b9bf166a4762d9930cd730f913fc4e61bcd8901ec27c424114c3e887ed372497f0c2cf85839b8443e76988b34}"
bind_address: "${LWM2M_BS_SECURITY_BIND_ADDRESS:0.0.0.0}"
bind_port: "${LWM2M_BS_SECURITY_BIND_PORT:5688}"
# Only Certificate_x509:
alias: "${LWM2M_KEYSTORE_ALIAS_BS:bootstrap}"
key_alias: "${LWM2M_BS_KEY_ALIAS:bootstrap}"
key_password: "${LWM2M_BS_KEY_PASSWORD:server_ks_password}"
security:
# Certificate_x509:
# To get helps about files format and how to generate it, see: https://github.com/eclipse/leshan/wiki/Credential-files-format
# Create new X509 Certificates: common/transport/lwm2m/src/main/resources/credentials/shell/lwM2M_credentials.sh
key_store_type: "${LWM2M_KEYSTORE_TYPE:JKS}"
# key_store_path_file: "${KEY_STORE_PATH_FILE:/common/transport/lwm2m/src/main/resources/credentials/serverKeyStore.jks"
key_store: "${LWM2M_KEY_STORE:lwm2mserver.jks}"
key_store_password: "${LWM2M_KEY_STORE_PASSWORD:server_ks_password}"
root_alias: "${LWM2M_SERVER_ROOT_CA:rootca}"
enable_gen_new_key_psk_rpk: "${ENABLE_GEN_NEW_KEY_PSK_RPK:false}"
key_store: "${LWM2M_KEYSTORE:lwm2mserver.jks}"
key_store_password: "${LWM2M_KEYSTORE_PASSWORD:server_ks_password}"
root_alias: "${LWM2M_SERVER_ROOT_CA_ALIAS:rootca}"
recommended_ciphers: "${LWM2M_RECOMMENDED_CIPHERS:false}"
recommended_supported_groups: "${LWM2M_RECOMMENDED_SUPPORTED_GROUPS:true}"
timeout: "${LWM2M_TIMEOUT:120000}"
recommended_ciphers: "${LWM2M_RECOMMENDED_CIPHERS:false}"
recommended_supported_groups: "${LWM2M_RECOMMENDED_SUPPORTED_GROUPS:true}"
uplink_pool_size: "${LWM2M_UPLINK_POOL_SIZE:10}"
downlink_pool_size: "${LWM2M_DOWNLINK_POOL_SIZE:10}"
ota_pool_size: "${LWM2M_OTA_POOL_SIZE:10}"
registration_store_pool_size: "${LWM2M_REGISTRATION_STORE_POOL_SIZE:100}"
clean_period_in_sec: "${LWM2M_CLEAN_PERIOD_IN_SEC:2}"
log_max_length: "${LWM2M_LOG_MAX_LENGTH:100}"
log_max_length: "${LWM2M_LOG_MAX_LENGTH:1024}"
# Use redis for Security and Registration stores
redis.enabled: "${LWM2M_REDIS_ENABLED:false}"
snmp:
@ -705,11 +702,14 @@ edges:
enabled: "${EDGES_RPC_SSL_ENABLED:false}"
cert: "${EDGES_RPC_SSL_CERT:certChainFile.pem}"
private_key: "${EDGES_RPC_SSL_PRIVATE_KEY:privateKeyFile.pem}"
max_inbound_message_size: "${EDGES_RPC_MAX_INBOUND_MESSAGE_SIZE:4194304}"
storage:
max_read_records_count: "${EDGES_STORAGE_MAX_READ_RECORDS_COUNT:50}"
no_read_records_sleep: "${EDGES_NO_READ_RECORDS_SLEEP:1000}"
sleep_between_batches: "${EDGES_SLEEP_BETWEEN_BATCHES:1000}"
scheduler_pool_size: "${EDGES_SCHEDULER_POOL_SIZE:4}"
scheduler_pool_size: "${EDGES_SCHEDULER_POOL_SIZE:1}"
send_scheduler_pool_size: "${EDGES_SEND_SCHEDULER_POOL_SIZE:1}"
grpc_callback_thread_pool_size: "${EDGES_GRPC_CALLBACK_POOL_SIZE:1}"
edge_events_ttl: "${EDGES_EDGE_EVENTS_TTL:0}"
state:
persistToTelemetry: "${EDGES_PERSIST_STATE_TO_TELEMETRY:false}"
@ -869,6 +869,7 @@ queue:
topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}"
poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}"
partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}"
consumer-per-partition: "${TB_QUEUE_RE_MAIN_CONSUMER_PER_PARTITION:true}"
pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:2000}"
submit-strategy:
type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
@ -885,6 +886,7 @@ queue:
topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}"
poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}"
partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}"
consumer-per-partition: "${TB_QUEUE_RE_HP_CONSUMER_PER_PARTITION:true}"
pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:2000}"
submit-strategy:
type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
@ -901,6 +903,7 @@ queue:
topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}"
poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}"
partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}"
consumer-per-partition: "${TB_QUEUE_RE_SQ_CONSUMER_PER_PARTITION:true}"
pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:2000}"
submit-strategy:
type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL

58
application/src/test/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessorTest.java

@ -0,0 +1,58 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.actors.device;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.common.util.LinkedHashMapRemoveEldest;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.device.DeviceService;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.mockito.BDDMockito.willReturn;
import static org.mockito.Mockito.mock;
public class DeviceActorMessageProcessorTest {
public static final long MAX_CONCURRENT_SESSIONS_PER_DEVICE = 10L;
ActorSystemContext systemContext;
DeviceService deviceService;
TenantId tenantId = TenantId.SYS_TENANT_ID;
DeviceId deviceId = DeviceId.fromString("78bf9b26-74ef-4af2-9cfb-ad6cf24ad2ec");
DeviceActorMessageProcessor processor;
@Before
public void setUp() {
systemContext = mock(ActorSystemContext.class);
deviceService = mock(DeviceService.class);
willReturn(MAX_CONCURRENT_SESSIONS_PER_DEVICE).given(systemContext).getMaxConcurrentSessionsPerDevice();
willReturn(deviceService).given(systemContext).getDeviceService();
processor = new DeviceActorMessageProcessor(systemContext, tenantId, deviceId);
}
@Test
public void givenSystemContext_whenNewInstance_thenVerifySessionMapMaxSize() {
assertThat(processor.sessions, instanceOf(LinkedHashMapRemoveEldest.class));
assertThat(processor.sessions.getMaxEntries(), is(MAX_CONCURRENT_SESSIONS_PER_DEVICE));
assertThat(processor.sessions.getRemovalConsumer(), notNullValue());
}
}

8
application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@ -352,10 +352,12 @@ public abstract class AbstractWebTest {
return mqttDeviceProfileTransportConfiguration;
}
protected ProtoTransportPayloadConfiguration createProtoTransportPayloadConfiguration(String deviceAttributesProtoSchema, String deviceTelemetryProtoSchema) {
protected ProtoTransportPayloadConfiguration createProtoTransportPayloadConfiguration(String attributesProtoSchema, String telemetryProtoSchema, String rpcRequestProtoSchema, String rpcResponseProtoSchema) {
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = new ProtoTransportPayloadConfiguration();
protoTransportPayloadConfiguration.setDeviceAttributesProtoSchema(deviceAttributesProtoSchema);
protoTransportPayloadConfiguration.setDeviceTelemetryProtoSchema(deviceTelemetryProtoSchema);
protoTransportPayloadConfiguration.setDeviceAttributesProtoSchema(attributesProtoSchema);
protoTransportPayloadConfiguration.setDeviceTelemetryProtoSchema(telemetryProtoSchema);
protoTransportPayloadConfiguration.setDeviceRpcRequestProtoSchema(rpcRequestProtoSchema);
protoTransportPayloadConfiguration.setDeviceRpcResponseProtoSchema(rpcResponseProtoSchema);
return protoTransportPayloadConfiguration;
}

228
application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java

@ -364,7 +364,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
"package schemavalidation;\n" +
"\n" +
"message SchemaValidationTest {\n" +
" int32 parameter = 1;\n" +
" optional int32 parameter = 1;\n" +
"}", "[Transport Configuration] invalid attributes proto schema provided! Schema options don't support!");
}
@ -377,7 +377,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
"package schemavalidation;\n" +
"\n" +
"message SchemaValidationTest {\n" +
" int32 parameter = 1;\n" +
" optional int32 parameter = 1;\n" +
"}", "[Transport Configuration] invalid attributes proto schema provided! Schema public imports don't support!");
}
@ -390,7 +390,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
"package schemavalidation;\n" +
"\n" +
"message SchemaValidationTest {\n" +
" int32 parameter = 1;\n" +
" optional int32 parameter = 1;\n" +
"}", "[Transport Configuration] invalid attributes proto schema provided! Schema imports don't support!");
}
@ -419,7 +419,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
"}\n" +
"\n" +
"message testMessage {\n" +
" int32 parameter = 1;\n" +
" optional int32 parameter = 1;\n" +
"}", "[Transport Configuration] invalid attributes proto schema provided! Enum definitions options are not supported!");
}
@ -444,7 +444,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
"\n" +
"message testMessage {\n" +
" option allow_alias = true;\n" +
" int32 parameter = 1;\n" +
" optional int32 parameter = 1;\n" +
"}", "[Transport Configuration] invalid attributes proto schema provided! Message definition options don't support!");
}
@ -479,8 +479,8 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
"\n" +
"message TestMessage {\n" +
" repeated group Result = 1 {\n" +
" string url = 2;\n" +
" string title = 3;\n" +
" optional string url = 2;\n" +
" optional string title = 3;\n" +
" repeated string snippets = 4;\n" +
" }\n" +
"}", "[Transport Configuration] invalid attributes proto schema provided! Message definition groups don't support!");
@ -513,15 +513,15 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
"message Outer {\n" +
" message MiddleAA {\n" +
" message Inner {\n" +
" int64 ival = 1;\n" +
" bool booly = 2;\n" +
" optional int64 ival = 1;\n" +
" optional bool booly = 2;\n" +
" }\n" +
" Inner inner = 1;\n" +
" }\n" +
" message MiddleBB {\n" +
" message Inner {\n" +
" int32 ival = 1;\n" +
" bool booly = 2;\n" +
" optional int32 ival = 1;\n" +
" optional bool booly = 2;\n" +
" }\n" +
" Inner inner = 1;\n" +
" }\n" +
@ -599,6 +599,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
"}\n" +
"\n" +
"message SampleMessage {\n" +
" optional int32 id = 1;\n" +
" oneof testOneOf {\n" +
" string name = 4;\n" +
" SubMessage subMessage = 9;\n" +
@ -616,7 +617,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
assertNotNull(sampleMsgDescriptor);
List<Descriptors.FieldDescriptor> fields = sampleMsgDescriptor.getFields();
assertEquals(2, fields.size());
assertEquals(3, fields.size());
DynamicMessage sampleMsg = sampleMsgBuilder
.setField(sampleMsgDescriptor.findFieldByName("name"), "Bob")
.build();
@ -634,8 +635,199 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
dynamicMsgToJson(sampleMsgDescriptor, sampleMsgWithOneOfSubMessage.toByteArray()));
}
@Test
public void testSaveProtoDeviceProfileWithInvalidTelemetrySchemaTsField() throws Exception {
testSaveDeviceProfileWithInvalidProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message PostTelemetry {\n" +
" int64 ts = 1;\n" +
" Values values = 2;\n" +
" \n" +
" message Values {\n" +
" string key1 = 3;\n" +
" bool key2 = 4;\n" +
" double key3 = 5;\n" +
" int32 key4 = 6;\n" +
" JsonObject key5 = 7;\n" +
" }\n" +
" \n" +
" message JsonObject {\n" +
" optional int32 someNumber = 8;\n" +
" repeated int32 someArray = 9;\n" +
" NestedJsonObject someNestedObject = 10;\n" +
" message NestedJsonObject {\n" +
" optional string key = 11;\n" +
" }\n" +
" }\n" +
"}", "[Transport Configuration] invalid telemetry proto schema provided! Field 'ts' has invalid label. Field 'ts' should have optional keyword!");
}
@Test
public void testSaveProtoDeviceProfileWithInvalidTelemetrySchemaTsDateType() throws Exception {
testSaveDeviceProfileWithInvalidProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message PostTelemetry {\n" +
" optional int32 ts = 1;\n" +
" Values values = 2;\n" +
" \n" +
" message Values {\n" +
" string key1 = 3;\n" +
" bool key2 = 4;\n" +
" double key3 = 5;\n" +
" int32 key4 = 6;\n" +
" JsonObject key5 = 7;\n" +
" }\n" +
" \n" +
" message JsonObject {\n" +
" optional int32 someNumber = 8;\n" +
" }\n" +
"}", "[Transport Configuration] invalid telemetry proto schema provided! Field 'ts' has invalid data type. Only int64 type is supported!");
}
@Test
public void testSaveProtoDeviceProfileWithInvalidTelemetrySchemaValuesDateType() throws Exception {
testSaveDeviceProfileWithInvalidProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message PostTelemetry {\n" +
" optional int64 ts = 1;\n" +
" string values = 2;\n" +
" \n" +
"}", "[Transport Configuration] invalid telemetry proto schema provided! Field 'values' has invalid data type. Only message type is supported!");
}
@Test
public void testSaveProtoDeviceProfileWithInvalidRpcRequestSchemaMethodDateType() throws Exception {
testSaveDeviceProfileWithInvalidRpcRequestProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" optional int32 method = 1;\n" +
" optional int32 requestId = 2;\n" +
" optional string params = 3;\n" +
" \n" +
"}", "[Transport Configuration] invalid rpc request proto schema provided! Field 'method' has invalid data type. Only string type is supported!");
}
@Test
public void testSaveProtoDeviceProfileWithInvalidRpcRequestSchemaRequestIdDateType() throws Exception {
testSaveDeviceProfileWithInvalidRpcRequestProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" optional string method = 1;\n" +
" optional int64 requestId = 2;\n" +
" optional string params = 3;\n" +
" \n" +
"}", "[Transport Configuration] invalid rpc request proto schema provided! Field 'requestId' has invalid data type. Only int32 type is supported!");
}
@Test
public void testSaveProtoDeviceProfileWithInvalidRpcRequestSchemaMethodLabel() throws Exception {
testSaveDeviceProfileWithInvalidRpcRequestProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" repeated string method = 1;\n" +
" optional int32 requestId = 2;\n" +
" optional string params = 3;\n" +
" \n" +
"}", "[Transport Configuration] invalid rpc request proto schema provided! Field 'method' has invalid label!");
}
@Test
public void testSaveProtoDeviceProfileWithInvalidRpcRequestSchemaRequestIdLabel() throws Exception {
testSaveDeviceProfileWithInvalidRpcRequestProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" optional string method = 1;\n" +
" repeated int32 requestId = 2;\n" +
" optional string params = 3;\n" +
" \n" +
"}", "[Transport Configuration] invalid rpc request proto schema provided! Field 'requestId' has invalid label!");
}
@Test
public void testSaveProtoDeviceProfileWithInvalidRpcRequestSchemaParamsLabel() throws Exception {
testSaveDeviceProfileWithInvalidRpcRequestProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" optional string method = 1;\n" +
" optional int32 requestId = 2;\n" +
" repeated string params = 3;\n" +
" \n" +
"}", "[Transport Configuration] invalid rpc request proto schema provided! Field 'params' has invalid label!");
}
@Test
public void testSaveProtoDeviceProfileWithInvalidRpcRequestSchemaFieldsCount() throws Exception {
testSaveDeviceProfileWithInvalidRpcRequestProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" optional int32 requestId = 2;\n" +
" optional string params = 3;\n" +
" \n" +
"}", "[Transport Configuration] invalid rpc request proto schema provided! RpcRequestMsg message should always contains 3 fields: method, requestId and params!");
}
@Test
public void testSaveProtoDeviceProfileWithInvalidRpcRequestSchemaFieldMethodIsNoSet() throws Exception {
testSaveDeviceProfileWithInvalidRpcRequestProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" optional string methodName = 1;\n" +
" optional int32 requestId = 2;\n" +
" optional string params = 3;\n" +
" \n" +
"}", "[Transport Configuration] invalid rpc request proto schema provided! Failed to get field descriptor for field: method!");
}
@Test
public void testSaveProtoDeviceProfileWithInvalidRpcRequestSchemaFieldRequestIdIsNotSet() throws Exception {
testSaveDeviceProfileWithInvalidRpcRequestProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" optional string method = 1;\n" +
" optional int32 requestIdentifier = 2;\n" +
" optional string params = 3;\n" +
" \n" +
"}", "[Transport Configuration] invalid rpc request proto schema provided! Failed to get field descriptor for field: requestId!");
}
@Test
public void testSaveProtoDeviceProfileWithInvalidRpcRequestSchemaFieldParamsIsNotSet() throws Exception {
testSaveDeviceProfileWithInvalidRpcRequestProtoSchema("syntax =\"proto3\";\n" +
"\n" +
"package schemavalidation;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" optional string method = 1;\n" +
" optional int32 requestId = 2;\n" +
" optional string parameters = 3;\n" +
" \n" +
"}", "[Transport Configuration] invalid rpc request proto schema provided! Failed to get field descriptor for field: params!");
}
private DeviceProfile testSaveDeviceProfileWithProtoPayloadType(String schema) throws Exception {
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema, null, null);
MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration);
DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration);
DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class);
@ -645,7 +837,15 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
}
private void testSaveDeviceProfileWithInvalidProtoSchema(String schema, String errorMsg) throws Exception {
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema, null, null);
MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration);
DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration);
doPost("/api/deviceProfile", deviceProfile).andExpect(status().isBadRequest())
.andExpect(statusReason(containsString(errorMsg)));
}
private void testSaveDeviceProfileWithInvalidRpcRequestProtoSchema(String schema, String errorMsg) throws Exception {
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema, schema, null);
MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration);
DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration);
doPost("/api/deviceProfile", deviceProfile).andExpect(status().isBadRequest())

37
application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java

@ -27,6 +27,7 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.id.DeviceProfileId;
@ -92,11 +93,12 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes
@Test
public void testSaveFirmware() throws Exception {
OtaPackageInfo firmwareInfo = new OtaPackageInfo();
SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest();
firmwareInfo.setDeviceProfileId(deviceProfileId);
firmwareInfo.setType(FIRMWARE);
firmwareInfo.setTitle(TITLE);
firmwareInfo.setVersion(VERSION);
firmwareInfo.setUsesUrl(false);
OtaPackageInfo savedFirmwareInfo = save(firmwareInfo);
@ -109,7 +111,7 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes
savedFirmwareInfo.setAdditionalInfo(JacksonUtil.newObjectNode());
save(savedFirmwareInfo);
save(new SaveOtaPackageInfoRequest(savedFirmwareInfo, false));
OtaPackageInfo foundFirmwareInfo = doGet("/api/otaPackage/info/" + savedFirmwareInfo.getId().getId().toString(), OtaPackageInfo.class);
Assert.assertEquals(foundFirmwareInfo.getTitle(), savedFirmwareInfo.getTitle());
@ -117,11 +119,12 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes
@Test
public void testSaveFirmwareData() throws Exception {
OtaPackageInfo firmwareInfo = new OtaPackageInfo();
SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest();
firmwareInfo.setDeviceProfileId(deviceProfileId);
firmwareInfo.setType(FIRMWARE);
firmwareInfo.setTitle(TITLE);
firmwareInfo.setVersion(VERSION);
firmwareInfo.setUsesUrl(false);
OtaPackageInfo savedFirmwareInfo = save(firmwareInfo);
@ -134,7 +137,7 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes
savedFirmwareInfo.setAdditionalInfo(JacksonUtil.newObjectNode());
save(savedFirmwareInfo);
save(new SaveOtaPackageInfoRequest(savedFirmwareInfo, false));
OtaPackageInfo foundFirmwareInfo = doGet("/api/otaPackage/info/" + savedFirmwareInfo.getId().getId().toString(), OtaPackageInfo.class);
Assert.assertEquals(foundFirmwareInfo.getTitle(), savedFirmwareInfo.getTitle());
@ -151,26 +154,31 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes
@Test
public void testUpdateFirmwareFromDifferentTenant() throws Exception {
OtaPackageInfo firmwareInfo = new OtaPackageInfo();
SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest();
firmwareInfo.setDeviceProfileId(deviceProfileId);
firmwareInfo.setType(FIRMWARE);
firmwareInfo.setTitle(TITLE);
firmwareInfo.setVersion(VERSION);
firmwareInfo.setUsesUrl(false);
OtaPackageInfo savedFirmwareInfo = save(firmwareInfo);
loginDifferentTenant();
doPost("/api/otaPackage", savedFirmwareInfo, OtaPackageInfo.class, status().isForbidden());
doPost("/api/otaPackage",
new SaveOtaPackageInfoRequest(savedFirmwareInfo, false),
OtaPackageInfo.class,
status().isForbidden());
deleteDifferentTenant();
}
@Test
public void testFindFirmwareInfoById() throws Exception {
OtaPackageInfo firmwareInfo = new OtaPackageInfo();
SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest();
firmwareInfo.setDeviceProfileId(deviceProfileId);
firmwareInfo.setType(FIRMWARE);
firmwareInfo.setTitle(TITLE);
firmwareInfo.setVersion(VERSION);
firmwareInfo.setUsesUrl(false);
OtaPackageInfo savedFirmwareInfo = save(firmwareInfo);
@ -181,11 +189,12 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes
@Test
public void testFindFirmwareById() throws Exception {
OtaPackageInfo firmwareInfo = new OtaPackageInfo();
SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest();
firmwareInfo.setDeviceProfileId(deviceProfileId);
firmwareInfo.setType(FIRMWARE);
firmwareInfo.setTitle(TITLE);
firmwareInfo.setVersion(VERSION);
firmwareInfo.setUsesUrl(false);
OtaPackageInfo savedFirmwareInfo = save(firmwareInfo);
@ -201,11 +210,12 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes
@Test
public void testDeleteFirmware() throws Exception {
OtaPackageInfo firmwareInfo = new OtaPackageInfo();
SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest();
firmwareInfo.setDeviceProfileId(deviceProfileId);
firmwareInfo.setType(FIRMWARE);
firmwareInfo.setTitle(TITLE);
firmwareInfo.setVersion(VERSION);
firmwareInfo.setUsesUrl(false);
OtaPackageInfo savedFirmwareInfo = save(firmwareInfo);
@ -220,11 +230,12 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes
public void testFindTenantFirmwares() throws Exception {
List<OtaPackageInfo> otaPackages = new ArrayList<>();
for (int i = 0; i < 165; i++) {
OtaPackageInfo firmwareInfo = new OtaPackageInfo();
SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest();
firmwareInfo.setDeviceProfileId(deviceProfileId);
firmwareInfo.setType(FIRMWARE);
firmwareInfo.setTitle(TITLE);
firmwareInfo.setVersion(VERSION + i);
firmwareInfo.setUsesUrl(false);
OtaPackageInfo savedFirmwareInfo = save(firmwareInfo);
@ -263,11 +274,12 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes
List<OtaPackageInfo> allOtaPackages = new ArrayList<>();
for (int i = 0; i < 165; i++) {
OtaPackageInfo firmwareInfo = new OtaPackageInfo();
SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest();
firmwareInfo.setDeviceProfileId(deviceProfileId);
firmwareInfo.setType(FIRMWARE);
firmwareInfo.setTitle(TITLE);
firmwareInfo.setVersion(VERSION + i);
firmwareInfo.setUsesUrl(false);
OtaPackageInfo savedFirmwareInfo = save(firmwareInfo);
@ -316,8 +328,7 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes
Assert.assertEquals(allOtaPackages, allLoadedOtaPackages);
}
private OtaPackageInfo save(OtaPackageInfo firmwareInfo) throws Exception {
private OtaPackageInfo save(SaveOtaPackageInfoRequest firmwareInfo) throws Exception {
return doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class);
}

1
application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java

@ -155,6 +155,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest {
doPost("/api/noauth/resetPasswordByEmail", resetPasswordByEmailRequest)
.andExpect(status().isOk());
Thread.sleep(1000);
doGet("/api/noauth/resetPassword?resetToken={resetToken}", TestMailService.currentResetPasswordToken)
.andExpect(status().isSeeOther())
.andExpect(header().string(HttpHeaders.LOCATION, "/login/resetPassword?resetToken=" + TestMailService.currentResetPasswordToken));

1
application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java

@ -29,6 +29,7 @@ import java.util.Arrays;
// "org.thingsboard.server.controller.sql.WebsocketApiSqlTest",
// "org.thingsboard.server.controller.sql.EntityQueryControllerSqlTest",
// "org.thingsboard.server.controller.sql.TbResourceControllerSqlTest",
// "org.thingsboard.server.controller.sql.DeviceProfileControllerSqlTest",
"org.thingsboard.server.controller.sql.*Test",
})
public class ControllerSqlTestSuite {

8
application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java

@ -216,7 +216,7 @@ public class EdgeImitator {
for (EntityDataProto entityData : downlinkMsg.getEntityDataList()) {
if (randomFailuresOnTimeseriesDownlink) {
if (getRandomBoolean()) {
result.add(Futures.immediateFailedFuture(new RuntimeException("Random failure")));
result.add(Futures.immediateFailedFuture(new RuntimeException("Random failure. This is expected error for edge test")));
} else {
result.add(saveDownlinkMsg(entityData));
}
@ -275,8 +275,8 @@ public class EdgeImitator {
private ListenableFuture<Void> saveDownlinkMsg(AbstractMessage message) {
if (!ignoredTypes.contains(message.getClass())) {
lock.lock();
try {
lock.lock();
downlinkMsgs.add(message);
} finally {
lock.unlock();
@ -312,8 +312,8 @@ public class EdgeImitator {
@SuppressWarnings("unchecked")
public <T extends AbstractMessage> Optional<T> findMessageByType(Class<T> tClass) {
Optional<T> result;
lock.lock();
try {
lock.lock();
result = (Optional<T>) downlinkMsgs.stream().filter(downlinkMsg -> downlinkMsg.getClass().isAssignableFrom(tClass)).findAny();
} finally {
lock.unlock();
@ -324,8 +324,8 @@ public class EdgeImitator {
@SuppressWarnings("unchecked")
public <T extends AbstractMessage> List<T> findAllMessagesByType(Class<T> tClass) {
List<T> result;
lock.lock();
try {
lock.lock();
result = (List<T>) downlinkMsgs.stream().filter(downlinkMsg -> downlinkMsg.getClass().isAssignableFrom(tClass)).collect(Collectors.toList());
} finally {
lock.unlock();

54
application/src/test/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaServiceTest.java

@ -0,0 +1,54 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import org.junit.Test;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.BDDMockito.willDoNothing;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
public class PsqlEntityDatabaseSchemaServiceTest {
@Test
public void givenPsqlDbSchemaService_whenCreateDatabaseSchema_thenVerifyPsqlIndexSpecificCall() throws Exception {
PsqlEntityDatabaseSchemaService service = spy(new PsqlEntityDatabaseSchemaService());
willDoNothing().given(service).executeQueryFromFile(anyString());
service.createDatabaseSchema();
verify(service, times(1)).createDatabaseIndexes();
verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_SQL);
verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_SQL);
verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL);
verify(service, times(3)).executeQueryFromFile(anyString());
}
@Test
public void givenPsqlDbSchemaService_whenCreateDatabaseIndexes_thenVerifyPsqlIndexSpecificCall() throws Exception {
PsqlEntityDatabaseSchemaService service = spy(new PsqlEntityDatabaseSchemaService());
willDoNothing().given(service).executeQueryFromFile(anyString());
service.createDatabaseIndexes();
verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_SQL);
verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL);
verify(service, times(2)).executeQueryFromFile(anyString());
}
}

2
application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java

@ -51,7 +51,7 @@ public class TestMailService {
currentResetPasswordToken = passwordResetLink.split("=")[1];
return null;
}
}).when(mailService).sendResetPasswordEmail(Mockito.anyString(), Mockito.anyString());
}).when(mailService).sendResetPasswordEmailAsync(Mockito.anyString(), Mockito.anyString());
return mailService;
}

83
application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java

@ -0,0 +1,83 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.state;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.service.queue.TbClusterService;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.mockito.BDDMockito.willReturn;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
@RunWith(MockitoJUnitRunner.class)
public class DefaultDeviceStateServiceTest {
@Mock
TenantService tenantService;
@Mock
DeviceService deviceService;
@Mock
AttributesService attributesService;
@Mock
TimeseriesService tsService;
@Mock
TbClusterService clusterService;
@Mock
PartitionService partitionService;
@Mock
DeviceStateData deviceStateDataMock;
DeviceId deviceId = DeviceId.fromString("00797a3b-7aeb-4b5b-b57a-c2a810d0f112");
DefaultDeviceStateService service;
@Before
public void setUp() {
service = spy(new DefaultDeviceStateService(tenantService, deviceService, attributesService, tsService, clusterService, partitionService));
}
@Test
public void givenDeviceIdFromDeviceStatesMap_whenGetOrFetchDeviceStateData_thenNoStackOverflow() {
service.deviceStates.put(deviceId, deviceStateDataMock);
DeviceStateData deviceStateData = service.getOrFetchDeviceStateData(deviceId);
assertThat(deviceStateData, is(deviceStateDataMock));
Mockito.verify(service, never()).fetchDeviceStateData(deviceId);
}
@Test
public void givenDeviceIdWithoutDeviceStateInMap_whenGetOrFetchDeviceStateData_thenFetchDeviceStateData() {
service.deviceStates.clear();
willReturn(deviceStateDataMock).given(service).fetchDeviceStateData(deviceId);
DeviceStateData deviceStateData = service.getOrFetchDeviceStateData(deviceId);
assertThat(deviceStateData, is(deviceStateDataMock));
Mockito.verify(service, times(1)).fetchDeviceStateData(deviceId);
}
}

50
application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java

@ -0,0 +1,50 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl;
import lombok.extern.slf4j.Slf4j;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.thingsboard.server.service.ttl.EventsCleanUpService.RANDOM_DELAY_INTERVAL_MS_EXPRESSION;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = EventsCleanUpServiceTest.class)
@Slf4j
public class EventsCleanUpServiceTest {
@Value(RANDOM_DELAY_INTERVAL_MS_EXPRESSION)
long randomDelayMs;
@Value("${sql.ttl.events.execution_interval_ms}")
long executionIntervalMs;
@Test
public void givenInterval_whenRandomDelay_ThenDelayInInterval() {
log.info("randomDelay {}", randomDelayMs);
log.info("executionIntervalMs {}", executionIntervalMs);
assertThat(executionIntervalMs, is(2220000L));
assertThat(randomDelayMs, greaterThanOrEqualTo(0L));
assertThat(randomDelayMs, lessThanOrEqualTo(executionIntervalMs));
}
}

34
application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java

@ -42,18 +42,18 @@ public abstract class AbstractTransportIntegrationTest extends AbstractControlle
"package test;\n" +
"\n" +
"message PostTelemetry {\n" +
" string key1 = 1;\n" +
" bool key2 = 2;\n" +
" double key3 = 3;\n" +
" int32 key4 = 4;\n" +
" optional string key1 = 1;\n" +
" optional bool key2 = 2;\n" +
" optional double key3 = 3;\n" +
" optional int32 key4 = 4;\n" +
" JsonObject key5 = 5;\n" +
"\n" +
" message JsonObject {\n" +
" int32 someNumber = 6;\n" +
" optional int32 someNumber = 6;\n" +
" repeated int32 someArray = 7;\n" +
" NestedJsonObject someNestedObject = 8;\n" +
" optional NestedJsonObject someNestedObject = 8;\n" +
" message NestedJsonObject {\n" +
" string key = 9;\n" +
" optional string key = 9;\n" +
" }\n" +
" }\n" +
"}";
@ -63,18 +63,18 @@ public abstract class AbstractTransportIntegrationTest extends AbstractControlle
"package test;\n" +
"\n" +
"message PostAttributes {\n" +
" string key1 = 1;\n" +
" bool key2 = 2;\n" +
" double key3 = 3;\n" +
" int32 key4 = 4;\n" +
" optional string key1 = 1;\n" +
" optional bool key2 = 2;\n" +
" optional double key3 = 3;\n" +
" optional int32 key4 = 4;\n" +
" JsonObject key5 = 5;\n" +
"\n" +
" message JsonObject {\n" +
" int32 someNumber = 6;\n" +
" optional int32 someNumber = 6;\n" +
" repeated int32 someArray = 7;\n" +
" NestedJsonObject someNestedObject = 8;\n" +
" message NestedJsonObject {\n" +
" string key = 9;\n" +
" optional string key = 9;\n" +
" }\n" +
" }\n" +
"}";
@ -83,16 +83,16 @@ public abstract class AbstractTransportIntegrationTest extends AbstractControlle
"package rpc;\n" +
"\n" +
"message RpcResponseMsg {\n" +
" string payload = 1;\n" +
" optional string payload = 1;\n" +
"}";
protected static final String DEVICE_RPC_REQUEST_PROTO_SCHEMA = "syntax =\"proto3\";\n" +
"package rpc;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" string method = 1;\n" +
" int32 requestId = 2;\n" +
" string params = 3;\n" +
" optional string method = 1;\n" +
" optional int32 requestId = 2;\n" +
" optional string params = 3;\n" +
"}";
protected Tenant savedTenant;

1
application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java

@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.msg.session.FeatureType;
import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;

8
application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcProtoIntegrationTest.java

@ -58,13 +58,13 @@ public abstract class AbstractCoapServerSideRpcProtoIntegrationTest extends Abst
"package rpc;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" string method = 1;\n" +
" int32 requestId = 2;\n" +
" optional string method = 1;\n" +
" optional int32 requestId = 2;\n" +
" Params params = 3;\n" +
"\n" +
" message Params {\n" +
" string pin = 1;\n" +
" int32 value = 2;\n" +
" optional string pin = 1;\n" +
" optional int32 value = 2;\n" +
" }\n" +
"}";

48
application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesIntegrationTest.java

@ -61,11 +61,14 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap
@Test
public void testPushAttributes() throws Exception {
List<String> expectedKeys = Arrays.asList("key1", "key2", "key3", "key4", "key5");
processAttributesTest(expectedKeys, PAYLOAD_VALUES_STR.getBytes());
processJsonPayloadAttributesTest(expectedKeys, PAYLOAD_VALUES_STR.getBytes());
}
protected void processAttributesTest(List<String> expectedKeys, byte[] payload) throws Exception {
log.warn("[testPushAttributes] Device: {}, Transport type: {}", savedDevice.getName(), savedDevice.getType());
protected void processJsonPayloadAttributesTest(List<String> expectedKeys, byte[] payload) throws Exception {
processAttributesTest(expectedKeys, payload, false);
}
protected void processAttributesTest(List<String> expectedKeys, byte[] payload, boolean presenceFieldsTest) throws Exception {
CoapClient client = getCoapClient(FeatureType.ATTRIBUTES);
postAttributes(client, payload);
@ -94,7 +97,11 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap
String getAttributesValuesUrl = getAttributesValuesUrl(deviceId, actualKeySet);
List<Map<String, Object>> values = doGetAsyncTyped(getAttributesValuesUrl, new TypeReference<>() {});
assertAttributesValues(values, expectedKeySet);
if (presenceFieldsTest) {
assertAttributesProtoValues(values, actualKeySet);
} else {
assertAttributesValues(values, actualKeySet);
}
String deleteAttributesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/CLIENT_SCOPE?keys=" + String.join(",", actualKeySet);
doDelete(deleteAttributesUrl);
}
@ -108,11 +115,11 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap
}
@SuppressWarnings("unchecked")
protected void assertAttributesValues(List<Map<String, Object>> deviceValues, Set<String> expectedKeySet) throws JsonProcessingException {
protected void assertAttributesValues(List<Map<String, Object>> deviceValues, Set<String> keySet) throws JsonProcessingException {
for (Map<String, Object> map : deviceValues) {
String key = (String) map.get("key");
Object value = map.get("value");
assertTrue(expectedKeySet.contains(key));
assertTrue(keySet.contains(key));
switch (key) {
case "key1":
assertEquals("value1", value);
@ -138,6 +145,35 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap
}
}
private void assertAttributesProtoValues(List<Map<String, Object>> values, Set<String> keySet) {
for (Map<String, Object> map : values) {
String key = (String) map.get("key");
Object value = map.get("value");
assertTrue(keySet.contains(key));
switch (key) {
case "key1":
assertEquals("", value);
break;
case "key2":
assertEquals(false, value);
break;
case "key3":
assertEquals(0.0, value);
break;
case "key4":
assertEquals(0, value);
break;
case "key5":
assertNotNull(value);
assertEquals(2, ((LinkedHashMap) value).size());
assertEquals(Arrays.asList(1, 2, 3), ((LinkedHashMap) value).get("someArray"));
LinkedHashMap<String, String> someNestedObject = (LinkedHashMap) ((LinkedHashMap) value).get("someNestedObject");
assertEquals("value", someNestedObject.get("key"));
break;
}
}
}
private String getAttributesValuesUrl(DeviceId deviceId, Set<String> actualKeySet) {
return "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/attributes/CLIENT_SCOPE?keys=" + String.join(",", actualKeySet);
}

47
application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesProtoIntegrationTest.java

@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadCo
import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration;
import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@ -48,7 +47,6 @@ public abstract class AbstractCoapAttributesProtoIntegrationTest extends Abstrac
@Test
public void testPushAttributes() throws Exception {
super.processBeforeTest("Test Post Attributes device Proto", CoapDeviceType.DEFAULT, TransportPayloadType.PROTOBUF);
List<String> expectedKeys = Arrays.asList("key1", "key2", "key3", "key4", "key5");
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
@ -87,7 +85,50 @@ public abstract class AbstractCoapAttributesProtoIntegrationTest extends Abstrac
.setField(postAttributesMsgDescriptor.findFieldByName("key4"), 4)
.setField(postAttributesMsgDescriptor.findFieldByName("key5"), jsonObject)
.build();
processAttributesTest(expectedKeys, postAttributesMsg.toByteArray());
processAttributesTest(Arrays.asList("key1", "key2", "key3", "key4", "key5"), postAttributesMsg.toByteArray(), false);
}
@Test
public void testPushAttributesWithExplicitPresenceProtoKeys() throws Exception {
super.processBeforeTest("Test Post Attributes device Proto", CoapDeviceType.DEFAULT, TransportPayloadType.PROTOBUF);
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration();
assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration);
DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA);
DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
assertNotNull(nestedJsonObjectBuilderDescriptor);
DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build();
DynamicMessage.Builder jsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject");
Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType();
assertNotNull(jsonObjectBuilderDescriptor);
DynamicMessage jsonObject = jsonObjectBuilder
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3)
.setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject)
.build();
DynamicMessage.Builder postAttributesBuilder = attributesSchema.newMessageBuilder("PostAttributes");
Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType();
assertNotNull(postAttributesMsgDescriptor);
DynamicMessage postAttributesMsg = postAttributesBuilder
.setField(postAttributesMsgDescriptor.findFieldByName("key1"), "")
.setField(postAttributesMsgDescriptor.findFieldByName("key2"), false)
.setField(postAttributesMsgDescriptor.findFieldByName("key3"), 0.0)
.setField(postAttributesMsgDescriptor.findFieldByName("key4"), 0)
.setField(postAttributesMsgDescriptor.findFieldByName("key5"), jsonObject)
.build();
processAttributesTest(Arrays.asList("key1", "key2", "key3", "key4", "key5"), postAttributesMsg.toByteArray(), true);
}
}

44
application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java

@ -56,18 +56,21 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap
@Test
public void testPushTelemetry() throws Exception {
processTestPostTelemetry(null, false);
processJsonPayloadTelemetryTest(PAYLOAD_VALUES_STR.getBytes(), false);
}
@Test
public void testPushTelemetryWithTs() throws Exception {
String payloadStr = "{\"ts\": 10000, \"values\": " + PAYLOAD_VALUES_STR + "}";
processTestPostTelemetry(payloadStr.getBytes(), true);
processJsonPayloadTelemetryTest(payloadStr.getBytes(), true);
}
protected void processTestPostTelemetry(byte[] payloadBytes, boolean withTs) throws Exception {
log.warn("[testPushTelemetry] Device: {}, Transport type: {}", savedDevice.getName(), savedDevice.getType());
protected void processJsonPayloadTelemetryTest(byte[] payloadBytes, boolean withTs) throws Exception {
List<String> expectedKeys = Arrays.asList("key1", "key2", "key3", "key4", "key5");
processTestPostTelemetry(payloadBytes, expectedKeys, withTs, false);
}
protected void processTestPostTelemetry(byte[] payloadBytes, List<String> expectedKeys, boolean withTs, boolean presenceFieldsTest) throws Exception {
CoapClient coapClient = getCoapClient(FeatureType.TELEMETRY);
postTelemetry(coapClient, payloadBytes);
@ -130,13 +133,14 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap
if (withTs) {
assertTs(values, expectedKeys, 10000, 0);
}
assertValues(values, 0);
if (presenceFieldsTest) {
assertExplicitProtoFieldValues(values);
} else {
assertValues(values, 0);
}
}
private void postTelemetry(CoapClient client, byte[] payload) throws IOException, ConnectorException {
if (payload == null) {
payload = PAYLOAD_VALUES_STR.getBytes();
}
CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON);
assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode());
}
@ -174,5 +178,29 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap
}
}
private void assertExplicitProtoFieldValues(Map<String, List<Map<String, Object>>> deviceValues) {
for (Map.Entry<String, List<Map<String, Object>>> entry : deviceValues.entrySet()) {
String key = entry.getKey();
List<Map<String, Object>> tsKv = entry.getValue();
String value = (String) tsKv.get(0).get("value");
switch (key) {
case "key1":
assertEquals("", value);
break;
case "key2":
assertEquals("false", value);
break;
case "key3":
assertEquals("0.0", value);
break;
case "key4":
assertEquals("0", value);
break;
case "key5":
assertEquals("{\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}", value);
break;
}
}
}
}

139
application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java

@ -32,6 +32,9 @@ import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportC
import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration;
import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration;
import java.util.Arrays;
import java.util.Collections;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@ -84,7 +87,7 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac
.setField(postTelemetryMsgDescriptor.findFieldByName("key4"), 4)
.setField(postTelemetryMsgDescriptor.findFieldByName("key5"), jsonObject)
.build();
processTestPostTelemetry(postTelemetryMsg.toByteArray(), false);
processTestPostTelemetry(postTelemetryMsg.toByteArray(), Arrays.asList("key1", "key2", "key3", "key4", "key5"), false, false);
}
@Test
@ -94,23 +97,23 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac
"package test;\n" +
"\n" +
"message PostTelemetry {\n" +
" int64 ts = 1;\n" +
" optional int64 ts = 1;\n" +
" Values values = 2;\n" +
" \n" +
" message Values {\n" +
" string key1 = 3;\n" +
" bool key2 = 4;\n" +
" double key3 = 5;\n" +
" int32 key4 = 6;\n" +
" optional string key1 = 3;\n" +
" optional bool key2 = 4;\n" +
" optional double key3 = 5;\n" +
" optional int32 key4 = 6;\n" +
" JsonObject key5 = 7;\n" +
" }\n" +
" \n" +
" message JsonObject {\n" +
" int32 someNumber = 8;\n" +
" optional int32 someNumber = 8;\n" +
" repeated int32 someArray = 9;\n" +
" NestedJsonObject someNestedObject = 10;\n" +
" message NestedJsonObject {\n" +
" string key = 11;\n" +
" optional string key = 11;\n" +
" }\n" +
" }\n" +
"}";
@ -164,7 +167,125 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac
.setField(postTelemetryMsgDescriptor.findFieldByName("values"), valuesMsg)
.build();
processTestPostTelemetry(postTelemetryMsg.toByteArray(), true);
processTestPostTelemetry(postTelemetryMsg.toByteArray(), Arrays.asList("key1", "key2", "key3", "key4", "key5"), true, false);
}
@Test
public void testPushTelemetryWithExplicitPresenceProtoKeys() throws Exception {
super.processBeforeTest("Test Post Telemetry device proto payload", CoapDeviceType.DEFAULT, TransportPayloadType.PROTOBUF);
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapDeviceProfileTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapDeviceProfileTransportConfiguration.getCoapDeviceTypeConfiguration();
assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration);
DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_TELEMETRY_PROTO_SCHEMA);
DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema");
DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
assertNotNull(nestedJsonObjectBuilderDescriptor);
DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build();
DynamicMessage.Builder jsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject");
Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType();
assertNotNull(jsonObjectBuilderDescriptor);
DynamicMessage jsonObject = jsonObjectBuilder
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3)
.setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject)
.build();
DynamicMessage.Builder postTelemetryBuilder = telemetrySchema.newMessageBuilder("PostTelemetry");
Descriptors.Descriptor postTelemetryMsgDescriptor = postTelemetryBuilder.getDescriptorForType();
assertNotNull(postTelemetryMsgDescriptor);
DynamicMessage postTelemetryMsg = postTelemetryBuilder
.setField(postTelemetryMsgDescriptor.findFieldByName("key1"), "")
.setField(postTelemetryMsgDescriptor.findFieldByName("key2"), false)
.setField(postTelemetryMsgDescriptor.findFieldByName("key3"), 0.0)
.setField(postTelemetryMsgDescriptor.findFieldByName("key4"), 0)
.setField(postTelemetryMsgDescriptor.findFieldByName("key5"), jsonObject)
.build();
processTestPostTelemetry(postTelemetryMsg.toByteArray(), Arrays.asList("key1", "key2", "key3", "key4", "key5"), false, true);
}
@Test
public void testPushTelemetryWithTsAndNoPresenceFields() throws Exception {
String schemaStr = "syntax =\"proto3\";\n" +
"\n" +
"package test;\n" +
"\n" +
"message PostTelemetry {\n" +
" optional int64 ts = 1;\n" +
" Values values = 2;\n" +
" \n" +
" message Values {\n" +
" string key1 = 3;\n" +
" bool key2 = 4;\n" +
" double key3 = 5;\n" +
" int32 key4 = 6;\n" +
" JsonObject key5 = 7;\n" +
" }\n" +
" \n" +
" message JsonObject {\n" +
" optional int32 someNumber = 8;\n" +
" repeated int32 someArray = 9;\n" +
" NestedJsonObject someNestedObject = 10;\n" +
" message NestedJsonObject {\n" +
" optional string key = 11;\n" +
" }\n" +
" }\n" +
"}";
super.processBeforeTest("Test Post Telemetry device proto payload", CoapDeviceType.DEFAULT, TransportPayloadType.PROTOBUF, schemaStr, null, null, null, null, null, DeviceProfileProvisionType.DISABLED);
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapDeviceProfileTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapDeviceProfileTransportConfiguration.getCoapDeviceTypeConfiguration();
assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration);
DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(schemaStr);
DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema");
DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
assertNotNull(nestedJsonObjectBuilderDescriptor);
DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build();
DynamicMessage.Builder jsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject");
Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType();
assertNotNull(jsonObjectBuilderDescriptor);
DynamicMessage jsonObject = jsonObjectBuilder
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3)
.setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject)
.build();
DynamicMessage.Builder valuesBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.Values");
Descriptors.Descriptor valuesDescriptor = valuesBuilder.getDescriptorForType();
assertNotNull(valuesDescriptor);
DynamicMessage valuesMsg = valuesBuilder
.setField(valuesDescriptor.findFieldByName("key5"), jsonObject)
.build();
DynamicMessage.Builder postTelemetryBuilder = telemetrySchema.newMessageBuilder("PostTelemetry");
Descriptors.Descriptor postTelemetryMsgDescriptor = postTelemetryBuilder.getDescriptorForType();
assertNotNull(postTelemetryMsgDescriptor);
DynamicMessage postTelemetryMsg = postTelemetryBuilder
.setField(postTelemetryMsgDescriptor.findFieldByName("ts"), 10000L)
.setField(postTelemetryMsgDescriptor.findFieldByName("values"), valuesMsg)
.build();
processTestPostTelemetry(postTelemetryMsg.toByteArray(), Arrays.asList("key1", "key2", "key3", "key4", "key5"), true, true);
}
}

146
application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java

@ -17,24 +17,46 @@ package org.thingsboard.server.transport.lwm2m;
import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.commons.io.IOUtils;
import org.eclipse.californium.core.network.config.NetworkConfig;
import org.eclipse.leshan.client.object.Security;
import org.eclipse.leshan.core.util.Hex;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.DeviceProfileProvisionType;
import org.thingsboard.server.common.data.DeviceProfileType;
import org.thingsboard.server.common.data.DeviceTransportType;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MClientCredentials;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration;
import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
import org.thingsboard.server.common.data.device.profile.DisabledDeviceProfileProvisionConfiguration;
import org.thingsboard.server.common.data.device.profile.Lwm2mDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataPageLink;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.SingleEntityFilter;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
import org.thingsboard.server.controller.AbstractWebsocketTest;
import org.thingsboard.server.controller.TbTestWebSocketClient;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.service.telemetry.cmd.TelemetryPluginCmdsWrapper;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataCmd;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate;
import org.thingsboard.server.service.telemetry.cmd.v2.LatestValueCmd;
import org.thingsboard.server.transport.lwm2m.client.LwM2MTestClient;
import org.thingsboard.server.transport.lwm2m.secure.credentials.LwM2MCredentials;
import java.io.IOException;
import java.io.InputStream;
@ -54,9 +76,16 @@ import java.security.spec.ECPrivateKeySpec;
import java.security.spec.ECPublicKeySpec;
import java.security.spec.KeySpec;
import java.util.Base64;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import static org.eclipse.leshan.client.object.Security.noSec;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE;
import static org.thingsboard.server.common.data.ota.OtaPackageType.SOFTWARE;
@DaoSqlTest
public class AbstractLwM2MIntegrationTest extends AbstractWebsocketTest {
@ -139,6 +168,15 @@ public class AbstractLwM2MIntegrationTest extends AbstractWebsocketTest {
// certificates trustedby the server (should contain rootCA)
protected final Certificate[] trustedCertificates = new Certificate[1];
protected static final int SECURE_PORT = 5686;
protected static final NetworkConfig SECURE_COAP_CONFIG = new NetworkConfig().setString("COAP_SECURE_PORT", Integer.toString(SECURE_PORT));
protected static final String ENDPOINT = "deviceAEndpoint";
protected static final String SECURE_URI = "coaps://localhost:" + SECURE_PORT;
protected static final int PORT = 5685;
protected static final Security SECURITY = noSec("coap://localhost:" + PORT, 123);
protected static final NetworkConfig COAP_CONFIG = new NetworkConfig().setString("COAP_PORT", Integer.toString(PORT));
public AbstractLwM2MIntegrationTest() {
// create client credentials
try {
@ -226,7 +264,7 @@ public class AbstractLwM2MIntegrationTest extends AbstractWebsocketTest {
executor = Executors.newScheduledThreadPool(10);
loginTenantAdmin();
String[] resources = new String[]{"1.xml", "2.xml", "3.xml"};
String[] resources = new String[]{"1.xml", "2.xml", "3.xml", "5.xml", "9.xml"};
for (String resourceName : resources) {
TbResource lwModel = new TbResource();
lwModel.setResourceType(ResourceType.LWM2M_MODEL);
@ -262,10 +300,116 @@ public class AbstractLwM2MIntegrationTest extends AbstractWebsocketTest {
Assert.assertNotNull(deviceProfile);
}
protected Device createDevice(LwM2MClientCredentials clientCredentials) throws Exception {
Device device = new Device();
device.setName("Device A");
device.setDeviceProfileId(deviceProfile.getId());
device.setTenantId(tenantId);
device = doPost("/api/device", device, Device.class);
Assert.assertNotNull(device);
DeviceCredentials deviceCredentials =
doGet("/api/device/" + device.getId().getId().toString() + "/credentials", DeviceCredentials.class);
Assert.assertEquals(device.getId(), deviceCredentials.getDeviceId());
deviceCredentials.setCredentialsType(DeviceCredentialsType.LWM2M_CREDENTIALS);
LwM2MCredentials credentials = new LwM2MCredentials();
credentials.setClient(clientCredentials);
deviceCredentials.setCredentialsValue(JacksonUtil.toString(credentials));
doPost("/api/device/credentials", deviceCredentials).andExpect(status().isOk());
return device;
}
protected OtaPackageInfo createFirmware() throws Exception {
String CHECKSUM = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a";
OtaPackageInfo firmwareInfo = new OtaPackageInfo();
firmwareInfo.setDeviceProfileId(deviceProfile.getId());
firmwareInfo.setType(FIRMWARE);
firmwareInfo.setTitle("My firmware");
firmwareInfo.setVersion("v1.0");
OtaPackageInfo savedFirmwareInfo = doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class);
MockMultipartFile testData = new MockMultipartFile("file", "filename.txt", "text/plain", new byte[]{1});
return savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, "SHA256");
}
protected OtaPackageInfo createSoftware() throws Exception {
String CHECKSUM = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a";
OtaPackageInfo swInfo = new OtaPackageInfo();
swInfo.setDeviceProfileId(deviceProfile.getId());
swInfo.setType(SOFTWARE);
swInfo.setTitle("My sw");
swInfo.setVersion("v1.0");
OtaPackageInfo savedFirmwareInfo = doPost("/api/otaPackage", swInfo, OtaPackageInfo.class);
MockMultipartFile testData = new MockMultipartFile("file", "filename.txt", "text/plain", new byte[]{1});
return savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, "SHA256");
}
protected OtaPackageInfo savaData(String urlTemplate, MockMultipartFile content, String... params) throws Exception {
MockMultipartHttpServletRequestBuilder postRequest = MockMvcRequestBuilders.multipart(urlTemplate, params);
postRequest.file(content);
setJwtToken(postRequest);
return readResponse(mockMvc.perform(postRequest).andExpect(status().isOk()), OtaPackageInfo.class);
}
@After
public void after() {
executor.shutdownNow();
wsClient.close();
}
public void basicTestConnectionObserveTelemetry(Security security,
LwM2MClientCredentials credentials,
NetworkConfig coapConfig,
String endpoint) throws Exception {
LwM2MTestClient client = null;
try {
createDeviceProfile(TRANSPORT_CONFIGURATION);
Device device = createDevice(credentials);
SingleEntityFilter sef = new SingleEntityFilter();
sef.setSingleEntity(device.getId());
LatestValueCmd latestCmd = new LatestValueCmd();
latestCmd.setKeys(Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "batteryLevel")));
EntityDataQuery edq = new EntityDataQuery(sef, new EntityDataPageLink(1, 0, null, null),
Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
EntityDataCmd cmd = new EntityDataCmd(1, edq, null, latestCmd, null);
TelemetryPluginCmdsWrapper wrapper = new TelemetryPluginCmdsWrapper();
wrapper.setEntityDataCmds(Collections.singletonList(cmd));
wsClient.send(mapper.writeValueAsString(wrapper));
wsClient.waitForReply();
wsClient.registerWaitForUpdate();
client = new LwM2MTestClient(executor, endpoint);
client.init(security, coapConfig);
String msg = wsClient.waitForUpdate();
EntityDataUpdate update = mapper.readValue(msg, EntityDataUpdate.class);
Assert.assertEquals(1, update.getCmdId());
List<EntityData> eData = update.getUpdate();
Assert.assertNotNull(eData);
Assert.assertEquals(1, eData.size());
Assert.assertEquals(device.getId(), eData.get(0).getEntityId());
Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES));
var tsValue = eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("batteryLevel");
Assert.assertEquals(42, Long.parseLong(tsValue.getValue()));
} finally {
if(client != null) {
client.destroy();
}
}
}
}

305
application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java

@ -15,175 +15,218 @@
*/
package org.thingsboard.server.transport.lwm2m;
import org.eclipse.californium.core.network.config.NetworkConfig;
import org.eclipse.leshan.client.object.Security;
import com.fasterxml.jackson.core.type.TypeReference;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.device.credentials.lwm2m.NoSecClientCredentials;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataPageLink;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.SingleEntityFilter;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
import org.thingsboard.server.service.telemetry.cmd.TelemetryPluginCmdsWrapper;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataCmd;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate;
import org.thingsboard.server.service.telemetry.cmd.v2.LatestValueCmd;
import org.thingsboard.server.transport.lwm2m.client.LwM2MTestClient;
import org.thingsboard.server.transport.lwm2m.secure.credentials.LwM2MCredentials;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import static org.eclipse.leshan.client.object.Security.noSec;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE;
import java.util.stream.Collectors;
import static org.thingsboard.rest.client.utils.RestJsonConverter.toTimeseries;
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.DOWNLOADED;
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.DOWNLOADING;
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.FAILED;
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.INITIATED;
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.QUEUED;
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.UPDATED;
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.UPDATING;
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.VERIFIED;
public class NoSecLwM2MIntegrationTest extends AbstractLwM2MIntegrationTest {
private final int PORT = 5685;
private final Security SECURITY = noSec("coap://localhost:" + PORT, 123);
private final NetworkConfig COAP_CONFIG = new NetworkConfig().setString("COAP_PORT", Integer.toString(PORT));
private final String ENDPOINT = "noSecEndpoint";
private Device createDevice() throws Exception {
Device device = new Device();
device.setName("Device A");
device.setDeviceProfileId(deviceProfile.getId());
device.setTenantId(tenantId);
device = doPost("/api/device", device, Device.class);
Assert.assertNotNull(device);
DeviceCredentials deviceCredentials =
doGet("/api/device/" + device.getId().getId().toString() + "/credentials", DeviceCredentials.class);
Assert.assertEquals(device.getId(), deviceCredentials.getDeviceId());
deviceCredentials.setCredentialsType(DeviceCredentialsType.LWM2M_CREDENTIALS);
LwM2MCredentials noSecCredentials = new LwM2MCredentials();
private final String OTA_TRANSPORT_CONFIGURATION = "{\n" +
" \"observeAttr\": {\n" +
" \"keyName\": {\n" +
" \"/5_1.0/0/3\": \"state\",\n" +
" \"/5_1.0/0/5\": \"updateResult\",\n" +
" \"/5_1.0/0/6\": \"pkgname\",\n" +
" \"/5_1.0/0/7\": \"pkgversion\",\n" +
" \"/5_1.0/0/9\": \"firmwareUpdateDeliveryMethod\",\n" +
" \"/9_1.0/0/0\": \"pkgname\",\n" +
" \"/9_1.0/0/1\": \"pkgversion\",\n" +
" \"/9_1.0/0/7\": \"updateState\",\n" +
" \"/9_1.0/0/9\": \"updateResult\"\n" +
" },\n" +
" \"observe\": [\n" +
" \"/5_1.0/0/3\",\n" +
" \"/5_1.0/0/5\",\n" +
" \"/5_1.0/0/6\",\n" +
" \"/5_1.0/0/7\",\n" +
" \"/5_1.0/0/9\",\n" +
" \"/9_1.0/0/0\",\n" +
" \"/9_1.0/0/1\",\n" +
" \"/9_1.0/0/7\",\n" +
" \"/9_1.0/0/9\"\n" +
" ],\n" +
" \"attribute\": [],\n" +
" \"telemetry\": [\n" +
" \"/5_1.0/0/3\",\n" +
" \"/5_1.0/0/5\",\n" +
" \"/5_1.0/0/6\",\n" +
" \"/5_1.0/0/7\",\n" +
" \"/5_1.0/0/9\",\n" +
" \"/9_1.0/0/0\",\n" +
" \"/9_1.0/0/1\",\n" +
" \"/9_1.0/0/7\",\n" +
" \"/9_1.0/0/9\"\n" +
" ],\n" +
" \"attributeLwm2m\": {}\n" +
" },\n" +
" \"bootstrap\": {\n" +
" \"servers\": {\n" +
" \"binding\": \"UQ\",\n" +
" \"shortId\": 123,\n" +
" \"lifetime\": 300,\n" +
" \"notifIfDisabled\": true,\n" +
" \"defaultMinPeriod\": 1\n" +
" },\n" +
" \"lwm2mServer\": {\n" +
" \"host\": \"localhost\",\n" +
" \"port\": 5685,\n" +
" \"serverId\": 123,\n" +
" \"securityMode\": \"NO_SEC\",\n" +
" \"serverPublicKey\": \"\",\n" +
" \"clientHoldOffTime\": 1,\n" +
" \"bootstrapServerAccountTimeout\": 0\n" +
" },\n" +
" \"bootstrapServer\": {\n" +
" \"host\": \"localhost\",\n" +
" \"port\": 5687,\n" +
" \"serverId\": 111,\n" +
" \"securityMode\": \"NO_SEC\",\n" +
" \"serverPublicKey\": \"\",\n" +
" \"clientHoldOffTime\": 1,\n" +
" \"bootstrapServerAccountTimeout\": 0\n" +
" }\n" +
" },\n" +
" \"clientLwM2mSettings\": {\n" +
" \"fwUpdateStrategy\": 1,\n" +
" \"swUpdateStrategy\": 1,\n" +
" \"clientOnlyObserveAfterConnect\": 1,\n" +
" \"powerMode\": \"PSM\",\n" +
" \"fwUpdateResource\": \"\",\n" +
" \"swUpdateResource\": \"\",\n" +
" \"compositeOperationsSupport\": false\n" +
" },\n" +
" \"type\": \"LWM2M\"\n" +
"}";
@Test
public void testConnectAndObserveTelemetry() throws Exception {
NoSecClientCredentials clientCredentials = new NoSecClientCredentials();
clientCredentials.setEndpoint(ENDPOINT);
noSecCredentials.setClient(clientCredentials);
deviceCredentials.setCredentialsValue(JacksonUtil.toString(noSecCredentials));
doPost("/api/device/credentials", deviceCredentials).andExpect(status().isOk());
return device;
super.basicTestConnectionObserveTelemetry(SECURITY, clientCredentials, COAP_CONFIG, ENDPOINT);
}
private OtaPackageInfo createFirmware() throws Exception {
String CHECKSUM = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a";
@Test
public void testFirmwareUpdateWithClientWithoutFirmwareInfo() throws Exception {
LwM2MTestClient client = null;
try {
createDeviceProfile(TRANSPORT_CONFIGURATION);
NoSecClientCredentials clientCredentials = new NoSecClientCredentials();
clientCredentials.setEndpoint(ENDPOINT);
Device device = createDevice(clientCredentials);
OtaPackageInfo firmwareInfo = new OtaPackageInfo();
firmwareInfo.setDeviceProfileId(deviceProfile.getId());
firmwareInfo.setType(FIRMWARE);
firmwareInfo.setTitle("My firmware");
firmwareInfo.setVersion("v1.0");
client = new LwM2MTestClient(executor, ENDPOINT);
client.init(SECURITY, COAP_CONFIG);
OtaPackageInfo savedFirmwareInfo = doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class);
Thread.sleep(1000);
MockMultipartFile testData = new MockMultipartFile("file", "filename.txt", "text/plain", new byte[]{1});
device.setFirmwareId(createFirmware().getId());
device = doPost("/api/device", device, Device.class);
return savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, "SHA256");
}
Thread.sleep(5000);
protected OtaPackageInfo savaData(String urlTemplate, MockMultipartFile content, String... params) throws Exception {
MockMultipartHttpServletRequestBuilder postRequest = MockMvcRequestBuilders.multipart(urlTemplate, params);
postRequest.file(content);
setJwtToken(postRequest);
return readResponse(mockMvc.perform(postRequest).andExpect(status().isOk()), OtaPackageInfo.class);
}
List<TsKvEntry> ts = toTimeseries(doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + device.getId().getId() + "/values/timeseries?keys=fw_state", new TypeReference<>() {
}));
@Test
public void testConnectAndObserveTelemetry() throws Exception {
createDeviceProfile(TRANSPORT_CONFIGURATION);
Device device = createDevice();
SingleEntityFilter sef = new SingleEntityFilter();
sef.setSingleEntity(device.getId());
LatestValueCmd latestCmd = new LatestValueCmd();
latestCmd.setKeys(Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "batteryLevel")));
EntityDataQuery edq = new EntityDataQuery(sef, new EntityDataPageLink(1, 0, null, null),
Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
EntityDataCmd cmd = new EntityDataCmd(1, edq, null, latestCmd, null);
TelemetryPluginCmdsWrapper wrapper = new TelemetryPluginCmdsWrapper();
wrapper.setEntityDataCmds(Collections.singletonList(cmd));
wsClient.send(mapper.writeValueAsString(wrapper));
wsClient.waitForReply();
wsClient.registerWaitForUpdate();
LwM2MTestClient client = new LwM2MTestClient(executor, ENDPOINT);
client.init(SECURITY, COAP_CONFIG);
String msg = wsClient.waitForUpdate();
EntityDataUpdate update = mapper.readValue(msg, EntityDataUpdate.class);
Assert.assertEquals(1, update.getCmdId());
List<EntityData> eData = update.getUpdate();
Assert.assertNotNull(eData);
Assert.assertEquals(1, eData.size());
Assert.assertEquals(device.getId(), eData.get(0).getEntityId());
Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES));
var tsValue = eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("batteryLevel");
Assert.assertEquals(42, Long.parseLong(tsValue.getValue()));
client.destroy();
List<OtaPackageUpdateStatus> statuses = ts.stream().map(KvEntry::getValueAsString).map(OtaPackageUpdateStatus::valueOf).collect(Collectors.toList());
List<OtaPackageUpdateStatus> expectedStatuses = Collections.singletonList(FAILED);
Assert.assertEquals(expectedStatuses, statuses);
} finally {
if(client != null) {
client.destroy();
}
}
}
@Test
public void testFirmwareUpdateWithClientWithoutFirmwareInfo() throws Exception {
createDeviceProfile(TRANSPORT_CONFIGURATION);
public void testFirmwareUpdateByObject5() throws Exception {
LwM2MTestClient client = null;
try {
createDeviceProfile(OTA_TRANSPORT_CONFIGURATION);
NoSecClientCredentials clientCredentials = new NoSecClientCredentials();
clientCredentials.setEndpoint("OTA_" + ENDPOINT);
Device device = createDevice(clientCredentials);
Device device = createDevice();
device.setFirmwareId(createFirmware().getId());
device = doPost("/api/device", device, Device.class);
Thread.sleep(1000);
OtaPackageInfo firmware = createFirmware();
client = new LwM2MTestClient(executor, "OTA_" + ENDPOINT);
client.init(SECURITY, COAP_CONFIG);
LwM2MTestClient client = new LwM2MTestClient(executor, ENDPOINT);
client.init(SECURITY, COAP_CONFIG);
Thread.sleep(3000);
Thread.sleep(1000);
List<TsKvEntry> ts = toTimeseries(doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + device.getId().getId() + "/values/timeseries?orderBy=ASC&keys=fw_state&startTs=0&endTs=" + System.currentTimeMillis(), new TypeReference<>() {
}));
device.setFirmwareId(firmware.getId());
List<OtaPackageUpdateStatus> statuses = ts.stream().sorted(Comparator.comparingLong(TsKvEntry::getTs)).map(KvEntry::getValueAsString).map(OtaPackageUpdateStatus::valueOf).collect(Collectors.toList());
device = doPost("/api/device", device, Device.class);
List<OtaPackageUpdateStatus> expectedStatuses = Arrays.asList(QUEUED, INITIATED, DOWNLOADING, DOWNLOADED, UPDATING, UPDATED);
Thread.sleep(1000);
Assert.assertEquals(expectedStatuses, statuses);
} finally {
if (client != null) {
client.destroy();
}
}
}
SingleEntityFilter sef = new SingleEntityFilter();
sef.setSingleEntity(device.getId());
LatestValueCmd latestCmd = new LatestValueCmd();
latestCmd.setKeys(Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "fw_state")));
EntityDataQuery edq = new EntityDataQuery(sef, new EntityDataPageLink(1, 0, null, null),
Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
@Test
public void testSoftwareUpdateByObject9() throws Exception {
LwM2MTestClient client = null;
try {
createDeviceProfile(OTA_TRANSPORT_CONFIGURATION);
NoSecClientCredentials clientCredentials = new NoSecClientCredentials();
clientCredentials.setEndpoint("OTA_" + ENDPOINT);
Device device = createDevice(clientCredentials);
EntityDataCmd cmd = new EntityDataCmd(1, edq, null, latestCmd, null);
TelemetryPluginCmdsWrapper wrapper = new TelemetryPluginCmdsWrapper();
wrapper.setEntityDataCmds(Collections.singletonList(cmd));
device.setSoftwareId(createSoftware().getId());
device = doPost("/api/device", device, Device.class);
wsClient.send(mapper.writeValueAsString(wrapper));
wsClient.waitForReply();
Thread.sleep(1000);
wsClient.registerWaitForUpdate();
client = new LwM2MTestClient(executor, "OTA_" + ENDPOINT);
client.init(SECURITY, COAP_CONFIG);
String msg = wsClient.waitForUpdate();
Thread.sleep(3000);
EntityDataUpdate update = mapper.readValue(msg, EntityDataUpdate.class);
Assert.assertEquals(1, update.getCmdId());
List<EntityData> eData = update.getUpdate();
Assert.assertNotNull(eData);
Assert.assertEquals(1, eData.size());
Assert.assertEquals(device.getId(), eData.get(0).getEntityId());
Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES));
var tsValue = eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("fw_state");
Assert.assertEquals("FAILED", tsValue.getValue());
client.destroy();
}
List<TsKvEntry> ts = toTimeseries(doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + device.getId().getId() + "/values/timeseries?orderBy=ASC&keys=sw_state&startTs=0&endTs=" + System.currentTimeMillis(), new TypeReference<>() {
}));
List<OtaPackageUpdateStatus> statuses = ts.stream().sorted(Comparator.comparingLong(TsKvEntry::getTs)).map(KvEntry::getValueAsString).map(OtaPackageUpdateStatus::valueOf).collect(Collectors.toList());
List<OtaPackageUpdateStatus> expectedStatuses = Arrays.asList(QUEUED, INITIATED, DOWNLOADING, DOWNLOADING, DOWNLOADING, DOWNLOADED, VERIFIED, UPDATED);
Assert.assertEquals(expectedStatuses, statuses);
} finally {
if (client != null) {
client.destroy();
}
}
}
}

43
application/src/test/java/org/thingsboard/server/transport/lwm2m/PskLwm2mIntegrationTest.java

@ -0,0 +1,43 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.transport.lwm2m;
import org.eclipse.leshan.client.object.Security;
import org.eclipse.leshan.core.util.Hex;
import org.junit.Test;
import org.thingsboard.server.common.data.device.credentials.lwm2m.PSKClientCredentials;
import java.nio.charset.StandardCharsets;
import static org.eclipse.leshan.client.object.Security.psk;
public class PskLwm2mIntegrationTest extends AbstractLwM2MIntegrationTest {
@Test
public void testConnectWithPSKAndObserveTelemetry() throws Exception {
String pskIdentity = "SOME_PSK_ID";
String pskKey = "73656372657450534b";
PSKClientCredentials clientCredentials = new PSKClientCredentials();
clientCredentials.setEndpoint(ENDPOINT);
clientCredentials.setKey(pskKey);
clientCredentials.setIdentity(pskIdentity);
Security security = psk(SECURE_URI,
123,
pskIdentity.getBytes(StandardCharsets.UTF_8),
Hex.decodeHex(pskKey.toCharArray()));
super.basicTestConnectionObserveTelemetry(security, clientCredentials, SECURE_COAP_CONFIG, ENDPOINT);
}
}

40
application/src/test/java/org/thingsboard/server/transport/lwm2m/RpkLwM2MIntegrationTest.java

@ -0,0 +1,40 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.transport.lwm2m;
import org.eclipse.leshan.client.object.Security;
import org.eclipse.leshan.core.util.Hex;
import org.junit.Test;
import org.thingsboard.server.common.data.device.credentials.lwm2m.RPKClientCredentials;
import static org.eclipse.leshan.client.object.Security.rpk;
public class RpkLwM2MIntegrationTest extends AbstractLwM2MIntegrationTest {
@Test
public void testConnectWithRPKAndObserveTelemetry() throws Exception {
RPKClientCredentials rpkClientCredentials = new RPKClientCredentials();
rpkClientCredentials.setEndpoint(ENDPOINT);
rpkClientCredentials.setKey(Hex.encodeHexString(clientPublicKey.getEncoded()));
Security security = rpk(SECURE_URI,
123,
clientPublicKey.getEncoded(),
clientPrivateKey.getEncoded(),
serverX509Cert.getPublicKey().getEncoded());
super.basicTestConnectionObserveTelemetry(security, rpkClientCredentials, SECURE_COAP_CONFIG, ENDPOINT);
}
}

137
application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java

@ -15,147 +15,40 @@
*/
package org.thingsboard.server.transport.lwm2m;
import org.eclipse.californium.core.network.config.NetworkConfig;
import org.eclipse.leshan.client.object.Security;
import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.device.credentials.lwm2m.X509ClientCredentials;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataPageLink;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.SingleEntityFilter;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
import org.thingsboard.server.common.transport.util.SslUtil;
import org.thingsboard.server.service.telemetry.cmd.TelemetryPluginCmdsWrapper;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataCmd;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate;
import org.thingsboard.server.service.telemetry.cmd.v2.LatestValueCmd;
import org.thingsboard.server.transport.lwm2m.client.LwM2MTestClient;
import org.thingsboard.server.transport.lwm2m.secure.credentials.LwM2MCredentials;
import java.util.Collections;
import java.util.List;
import static org.eclipse.leshan.client.object.Security.x509;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
public class X509LwM2MIntegrationTest extends AbstractLwM2MIntegrationTest {
private final int port = 5686;
private final NetworkConfig coapConfig = new NetworkConfig().setString("COAP_SECURE_PORT", Integer.toString(port));
private final String endpoint = "deviceAEndpoint";
private final String serverUri = "coaps://localhost:" + port;
private Device createDevice(X509ClientCredentials clientCredentials) throws Exception {
Device device = new Device();
device.setName("Device A");
device.setDeviceProfileId(deviceProfile.getId());
device.setTenantId(tenantId);
device = doPost("/api/device", device, Device.class);
Assert.assertNotNull(device);
DeviceCredentials deviceCredentials =
doGet("/api/device/" + device.getId().getId().toString() + "/credentials", DeviceCredentials.class);
Assert.assertEquals(device.getId(), deviceCredentials.getDeviceId());
deviceCredentials.setCredentialsType(DeviceCredentialsType.LWM2M_CREDENTIALS);
LwM2MCredentials credentials = new LwM2MCredentials();
credentials.setClient(clientCredentials);
deviceCredentials.setCredentialsValue(JacksonUtil.toString(credentials));
doPost("/api/device/credentials", deviceCredentials).andExpect(status().isOk());
return device;
}
//TODO: use different endpoints to isolate tests.
@Ignore()
@Test
public void testConnectAndObserveTelemetry() throws Exception {
createDeviceProfile(TRANSPORT_CONFIGURATION);
X509ClientCredentials credentials = new X509ClientCredentials();
credentials.setEndpoint(endpoint+1);
Device device = createDevice(credentials);
SingleEntityFilter sef = new SingleEntityFilter();
sef.setSingleEntity(device.getId());
LatestValueCmd latestCmd = new LatestValueCmd();
latestCmd.setKeys(Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "batteryLevel")));
EntityDataQuery edq = new EntityDataQuery(sef, new EntityDataPageLink(1, 0, null, null),
Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
EntityDataCmd cmd = new EntityDataCmd(1, edq, null, latestCmd, null);
TelemetryPluginCmdsWrapper wrapper = new TelemetryPluginCmdsWrapper();
wrapper.setEntityDataCmds(Collections.singletonList(cmd));
wsClient.send(mapper.writeValueAsString(wrapper));
wsClient.waitForReply();
wsClient.registerWaitForUpdate();
LwM2MTestClient client = new LwM2MTestClient(executor, endpoint+1);
Security security = x509(serverUri, 123, clientX509Cert.getEncoded(), clientPrivateKeyFromCert.getEncoded(), serverX509Cert.getEncoded());
client.init(security, coapConfig);
String msg = wsClient.waitForUpdate();
EntityDataUpdate update = mapper.readValue(msg, EntityDataUpdate.class);
Assert.assertEquals(1, update.getCmdId());
List<EntityData> eData = update.getUpdate();
Assert.assertNotNull(eData);
Assert.assertEquals(1, eData.size());
Assert.assertEquals(device.getId(), eData.get(0).getEntityId());
Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES));
var tsValue = eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("batteryLevel");
Assert.assertEquals(42, Long.parseLong(tsValue.getValue()));
client.destroy();
credentials.setEndpoint(ENDPOINT);
Security security = x509(SECURE_URI,
123,
clientX509Cert.getEncoded(),
clientPrivateKeyFromCert.getEncoded(),
serverX509Cert.getEncoded());
super.basicTestConnectionObserveTelemetry(security, credentials, SECURE_COAP_CONFIG, ENDPOINT);
}
@Ignore //See LwM2mClientContextImpl.unregister
@Test
public void testConnectWithCertAndObserveTelemetry() throws Exception {
createDeviceProfile(TRANSPORT_CONFIGURATION);
X509ClientCredentials credentials = new X509ClientCredentials();
credentials.setEndpoint(endpoint);
credentials.setEndpoint(ENDPOINT);
credentials.setCert(SslUtil.getCertificateString(clientX509CertNotTrusted));
Device device = createDevice(credentials);
SingleEntityFilter sef = new SingleEntityFilter();
sef.setSingleEntity(device.getId());
LatestValueCmd latestCmd = new LatestValueCmd();
latestCmd.setKeys(Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "batteryLevel")));
EntityDataQuery edq = new EntityDataQuery(sef, new EntityDataPageLink(1, 0, null, null),
Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
EntityDataCmd cmd = new EntityDataCmd(1, edq, null, latestCmd, null);
TelemetryPluginCmdsWrapper wrapper = new TelemetryPluginCmdsWrapper();
wrapper.setEntityDataCmds(Collections.singletonList(cmd));
wsClient.send(mapper.writeValueAsString(wrapper));
wsClient.waitForReply();
wsClient.registerWaitForUpdate();
LwM2MTestClient client = new LwM2MTestClient(executor, endpoint);
Security security = x509(serverUri, 123, clientX509CertNotTrusted.getEncoded(), clientPrivateKeyFromCert.getEncoded(), serverX509Cert.getEncoded());
client.init(security, coapConfig);
String msg = wsClient.waitForUpdate();
EntityDataUpdate update = mapper.readValue(msg, EntityDataUpdate.class);
Assert.assertEquals(1, update.getCmdId());
List<EntityData> eData = update.getUpdate();
Assert.assertNotNull(eData);
Assert.assertEquals(1, eData.size());
Assert.assertEquals(device.getId(), eData.get(0).getEntityId());
Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES));
var tsValue = eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("batteryLevel");
Assert.assertEquals(42, Long.parseLong(tsValue.getValue()));
client.destroy();
Security security = x509(SECURE_URI,
123,
clientX509CertNotTrusted.getEncoded(),
clientPrivateKeyFromCert.getEncoded(),
serverX509Cert.getEncoded());
super.basicTestConnectionObserveTelemetry(security, credentials, SECURE_COAP_CONFIG, ENDPOINT);
}
}

155
application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java

@ -0,0 +1,155 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.transport.lwm2m.client;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.leshan.client.resource.BaseInstanceEnabler;
import org.eclipse.leshan.client.servers.ServerIdentity;
import org.eclipse.leshan.core.model.ObjectModel;
import org.eclipse.leshan.core.node.LwM2mResource;
import org.eclipse.leshan.core.response.ExecuteResponse;
import org.eclipse.leshan.core.response.ReadResponse;
import org.eclipse.leshan.core.response.WriteResponse;
import javax.security.auth.Destroyable;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
public class FwLwM2MDevice extends BaseInstanceEnabler implements Destroyable {
private static final List<Integer> supportedResources = Arrays.asList(0, 1, 2, 3, 5, 6, 7, 9);
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
private final AtomicInteger state = new AtomicInteger(0);
private final AtomicInteger updateResult = new AtomicInteger(0);
@Override
public ReadResponse read(ServerIdentity identity, int resourceId) {
if (!identity.isSystem())
log.info("Read on Device resource /{}/{}/{}", getModel().id, getId(), resourceId);
switch (resourceId) {
case 3:
return ReadResponse.success(resourceId, getState());
case 5:
return ReadResponse.success(resourceId, getUpdateResult());
case 6:
return ReadResponse.success(resourceId, getPkgName());
case 7:
return ReadResponse.success(resourceId, getPkgVersion());
case 9:
return ReadResponse.success(resourceId, getFirmwareUpdateDeliveryMethod());
default:
return super.read(identity, resourceId);
}
}
@Override
public ExecuteResponse execute(ServerIdentity identity, int resourceId, String params) {
String withParams = null;
if (params != null && params.length() != 0) {
withParams = " with params " + params;
}
log.info("Execute on Device resource /{}/{}/{} {}", getModel().id, getId(), resourceId, withParams != null ? withParams : "");
switch (resourceId) {
case 2:
startUpdating();
return ExecuteResponse.success();
default:
return super.execute(identity, resourceId, params);
}
}
@Override
public WriteResponse write(ServerIdentity identity, boolean replace, int resourceId, LwM2mResource value) {
log.info("Write on Device resource /{}/{}/{}", getModel().id, getId(), resourceId);
switch (resourceId) {
case 0:
startDownloading();
return WriteResponse.success();
case 1:
startDownloading();
return WriteResponse.success();
default:
return super.write(identity, replace, resourceId, value);
}
}
private int getState() {
return state.get();
}
private int getUpdateResult() {
return updateResult.get();
}
private String getPkgName() {
return "firmware";
}
private String getPkgVersion() {
return "1.0.0";
}
private int getFirmwareUpdateDeliveryMethod() {
return 1;
}
@Override
public List<Integer> getAvailableResourceIds(ObjectModel model) {
return supportedResources;
}
@Override
public void destroy() {
scheduler.shutdownNow();
}
private void startDownloading() {
scheduler.schedule(() -> {
try {
state.set(1);
fireResourcesChange(3);
Thread.sleep(100);
state.set(2);
fireResourcesChange(3);
} catch (Exception e) {
}
}, 100, TimeUnit.MILLISECONDS);
}
private void startUpdating() {
scheduler.schedule(() -> {
try {
state.set(3);
fireResourcesChange(3);
Thread.sleep(100);
updateResult.set(1);
fireResourcesChange(5);
} catch (Exception e) {
}
}, 100, TimeUnit.MILLISECONDS);
}
}

6
application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java

@ -53,8 +53,10 @@ import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
import static org.eclipse.leshan.core.LwM2mId.DEVICE;
import static org.eclipse.leshan.core.LwM2mId.FIRMWARE;
import static org.eclipse.leshan.core.LwM2mId.SECURITY;
import static org.eclipse.leshan.core.LwM2mId.SERVER;
import static org.eclipse.leshan.core.LwM2mId.SOFTWARE_MANAGEMENT;
@Slf4j
@Data
@ -65,7 +67,7 @@ public class LwM2MTestClient {
private LeshanClient client;
public void init(Security security, NetworkConfig coapConfig) throws InvalidDDFFileException, IOException {
String[] resources = new String[]{"0.xml", "1.xml", "2.xml", "3.xml"};
String[] resources = new String[]{"0.xml", "1.xml", "2.xml", "3.xml", "5.xml", "9.xml"};
List<ObjectModel> models = new ArrayList<>();
for (String resourceName : resources) {
models.addAll(ObjectLoader.loadDdfFile(LwM2MTestClient.class.getClassLoader().getResourceAsStream("lwm2m/" + resourceName), resourceName));
@ -75,6 +77,8 @@ public class LwM2MTestClient {
initializer.setInstancesForObject(SECURITY, security);
initializer.setInstancesForObject(SERVER, new Server(123, 300));
initializer.setInstancesForObject(DEVICE, new SimpleLwM2MDevice());
initializer.setInstancesForObject(FIRMWARE, new FwLwM2MDevice());
initializer.setInstancesForObject(SOFTWARE_MANAGEMENT, new SwLwM2MDevice());
initializer.setClassForObject(LwM2mId.ACCESS_CONTROL, DummyInstanceEnabler.class);
DtlsConnectorConfig.Builder dtlsConfig = new DtlsConnectorConfig.Builder();

54
application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SimpleLwM2MDevice.java

@ -45,74 +45,74 @@ public class SimpleLwM2MDevice extends BaseInstanceEnabler implements Destroyabl
);
@Override
public ReadResponse read(ServerIdentity identity, int resourceid) {
public ReadResponse read(ServerIdentity identity, int resourceId) {
if (!identity.isSystem())
log.info("Read on Device resource /{}/{}/{}", getModel().id, getId(), resourceid);
switch (resourceid) {
log.info("Read on Device resource /{}/{}/{}", getModel().id, getId(), resourceId);
switch (resourceId) {
case 0:
return ReadResponse.success(resourceid, getManufacturer());
return ReadResponse.success(resourceId, getManufacturer());
case 1:
return ReadResponse.success(resourceid, getModelNumber());
return ReadResponse.success(resourceId, getModelNumber());
case 2:
return ReadResponse.success(resourceid, getSerialNumber());
return ReadResponse.success(resourceId, getSerialNumber());
case 3:
return ReadResponse.success(resourceid, getFirmwareVersion());
return ReadResponse.success(resourceId, getFirmwareVersion());
case 9:
return ReadResponse.success(resourceid, getBatteryLevel());
return ReadResponse.success(resourceId, getBatteryLevel());
case 10:
return ReadResponse.success(resourceid, getMemoryFree());
return ReadResponse.success(resourceId, getMemoryFree());
case 11:
Map<Integer, Long> errorCodes = new HashMap<>();
errorCodes.put(0, getErrorCode());
return ReadResponse.success(resourceid, errorCodes, ResourceModel.Type.INTEGER);
return ReadResponse.success(resourceId, errorCodes, ResourceModel.Type.INTEGER);
case 14:
return ReadResponse.success(resourceid, getUtcOffset());
return ReadResponse.success(resourceId, getUtcOffset());
case 15:
return ReadResponse.success(resourceid, getTimezone());
return ReadResponse.success(resourceId, getTimezone());
case 16:
return ReadResponse.success(resourceid, getSupportedBinding());
return ReadResponse.success(resourceId, getSupportedBinding());
case 17:
return ReadResponse.success(resourceid, getDeviceType());
return ReadResponse.success(resourceId, getDeviceType());
case 18:
return ReadResponse.success(resourceid, getHardwareVersion());
return ReadResponse.success(resourceId, getHardwareVersion());
case 19:
return ReadResponse.success(resourceid, getSoftwareVersion());
return ReadResponse.success(resourceId, getSoftwareVersion());
case 20:
return ReadResponse.success(resourceid, getBatteryStatus());
return ReadResponse.success(resourceId, getBatteryStatus());
case 21:
return ReadResponse.success(resourceid, getMemoryTotal());
return ReadResponse.success(resourceId, getMemoryTotal());
default:
return super.read(identity, resourceid);
return super.read(identity, resourceId);
}
}
@Override
public ExecuteResponse execute(ServerIdentity identity, int resourceid, String params) {
public ExecuteResponse execute(ServerIdentity identity, int resourceId, String params) {
String withParams = null;
if (params != null && params.length() != 0) {
withParams = " with params " + params;
}
log.info("Execute on Device resource /{}/{}/{} {}", getModel().id, getId(), resourceid, withParams != null ? withParams : "");
log.info("Execute on Device resource /{}/{}/{} {}", getModel().id, getId(), resourceId, withParams != null ? withParams : "");
return ExecuteResponse.success();
}
@Override
public WriteResponse write(ServerIdentity identity, boolean replace, int resourceid, LwM2mResource value) {
log.info("Write on Device resource /{}/{}/{}", getModel().id, getId(), resourceid);
public WriteResponse write(ServerIdentity identity, boolean replace, int resourceId, LwM2mResource value) {
log.info("Write on Device resource /{}/{}/{}", getModel().id, getId(), resourceId);
switch (resourceid) {
switch (resourceId) {
case 13:
return WriteResponse.notFound();
case 14:
setUtcOffset((String) value.getValue());
fireResourcesChange(resourceid);
fireResourcesChange(resourceId);
return WriteResponse.success();
case 15:
setTimezone((String) value.getValue());
fireResourcesChange(resourceid);
fireResourcesChange(resourceId);
return WriteResponse.success();
default:
return super.write(identity, replace, resourceid, value);
return super.write(identity, replace, resourceId, value);
}
}

155
application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SwLwM2MDevice.java

@ -0,0 +1,155 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.transport.lwm2m.client;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.leshan.client.resource.BaseInstanceEnabler;
import org.eclipse.leshan.client.servers.ServerIdentity;
import org.eclipse.leshan.core.model.ObjectModel;
import org.eclipse.leshan.core.node.LwM2mResource;
import org.eclipse.leshan.core.response.ExecuteResponse;
import org.eclipse.leshan.core.response.ReadResponse;
import org.eclipse.leshan.core.response.WriteResponse;
import javax.security.auth.Destroyable;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
public class SwLwM2MDevice extends BaseInstanceEnabler implements Destroyable {
private static final List<Integer> supportedResources = Arrays.asList(0, 1, 2, 3, 4, 6, 7, 9);
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
private final AtomicInteger state = new AtomicInteger(0);
private final AtomicInteger updateResult = new AtomicInteger(0);
@Override
public ReadResponse read(ServerIdentity identity, int resourceId) {
if (!identity.isSystem())
log.info("Read on Device resource /{}/{}/{}", getModel().id, getId(), resourceId);
switch (resourceId) {
case 0:
return ReadResponse.success(resourceId, getPkgName());
case 1:
return ReadResponse.success(resourceId, getPkgVersion());
case 7:
return ReadResponse.success(resourceId, getUpdateState());
case 9:
return ReadResponse.success(resourceId, getUpdateResult());
default:
return super.read(identity, resourceId);
}
}
@Override
public ExecuteResponse execute(ServerIdentity identity, int resourceId, String params) {
String withParams = null;
if (params != null && params.length() != 0) {
withParams = " with params " + params;
}
log.info("Execute on Device resource /{}/{}/{} {}", getModel().id, getId(), resourceId, withParams != null ? withParams : "");
switch (resourceId) {
case 4:
startUpdating();
return ExecuteResponse.success();
case 6:
return ExecuteResponse.success();
default:
return super.execute(identity, resourceId, params);
}
}
@Override
public WriteResponse write(ServerIdentity identity, boolean replace, int resourceId, LwM2mResource value) {
log.info("Write on Device resource /{}/{}/{}", getModel().id, getId(), resourceId);
switch (resourceId) {
case 2:
startDownloading();
return WriteResponse.success();
case 3:
startDownloading();
return WriteResponse.success();
default:
return super.write(identity, replace, resourceId, value);
}
}
private int getUpdateState() {
return state.get();
}
private int getUpdateResult() {
return updateResult.get();
}
private String getPkgName() {
return "software";
}
private String getPkgVersion() {
return "1.0.0";
}
@Override
public List<Integer> getAvailableResourceIds(ObjectModel model) {
return supportedResources;
}
@Override
public void destroy() {
scheduler.shutdownNow();
}
private void startDownloading() {
scheduler.schedule(() -> {
try {
state.set(1);
updateResult.set(1);
fireResourcesChange(7, 9);
Thread.sleep(100);
state.set(2);
fireResourcesChange(7);
Thread.sleep(100);
state.set(3);
fireResourcesChange(7);
Thread.sleep(100);
updateResult.set(3);
fireResourcesChange(9);
} catch (Exception e) {
}
}, 100, TimeUnit.MILLISECONDS);
}
private void startUpdating() {
scheduler.schedule(() -> {
state.set(4);
updateResult.set(2);
fireResourcesChange(7, 9);
}, 100, TimeUnit.MILLISECONDS);
}
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save