Browse Source

Merge branch 'master' into master

pull/10748/head
cogic 2 years ago
committed by GitHub
parent
commit
c3bab5bb02
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 5
      application/src/main/data/json/system/widget_bundles/charts.json
  2. 2
      application/src/main/data/json/system/widget_types/alarms_table.json
  3. 2
      application/src/main/data/json/system/widget_types/asset_admin_table.json
  4. 2
      application/src/main/data/json/system/widget_types/device_admin_table.json
  5. 2
      application/src/main/data/json/system/widget_types/entities_hierarchy.json
  6. 2
      application/src/main/data/json/system/widget_types/entities_table.json
  7. 33
      application/src/main/data/json/system/widget_types/radar.json
  8. 25
      application/src/main/data/json/system/widget_types/radar_deprecated.json
  9. 2
      application/src/main/data/json/system/widget_types/timeseries_table.json
  10. 17
      application/src/main/data/json/tenant/dashboards/gateways.json
  11. 25
      application/src/main/data/upgrade/3.6.4/schema_update.sql
  12. 4
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java
  13. 28
      application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java
  14. 3
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  15. 5
      application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java
  16. 2
      application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java
  17. 2
      application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java
  18. 135
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  19. 2
      application/src/main/java/org/thingsboard/server/service/install/EntityDatabaseSchemaService.java
  20. 10
      application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java
  21. 7
      application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java
  22. 2
      application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java
  23. 1
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java
  24. 82
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
  25. 43
      application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java
  26. 37
      application/src/main/java/org/thingsboard/server/service/mail/MailSenderInternalExecutorService.java
  27. 1
      application/src/main/java/org/thingsboard/server/service/mail/TbMailContextComponent.java
  28. 3
      application/src/main/java/org/thingsboard/server/service/notification/channels/MobileAppNotificationChannel.java
  29. 416
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  30. 41
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  31. 322
      application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java
  32. 131
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  33. 8
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java
  34. 7
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java
  35. 25
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java
  36. 290
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java
  37. 80
      application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsService.java
  38. 7
      application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java
  39. 4
      application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsService.java
  40. 72
      application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java
  41. 6
      application/src/main/java/org/thingsboard/server/service/security/model/token/OAuth2AppTokenFactory.java
  42. 28
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java
  43. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java
  44. 21
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
  45. 10
      application/src/main/resources/thingsboard.yml
  46. 17
      application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
  47. 4
      application/src/test/java/org/thingsboard/server/controller/AdminControllerTest.java
  48. 7
      application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java
  49. 6
      application/src/test/java/org/thingsboard/server/controller/AuthControllerTest.java
  50. 27
      application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java
  51. 38
      application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java
  52. 13
      application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java
  53. 2
      application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java
  54. 62
      application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java
  55. 35
      application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java
  56. 17
      application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java
  57. 5
      common/actor/pom.xml
  58. 69
      common/actor/src/test/java/org/thingsboard/server/actors/ActorSystemTest.java
  59. 9
      common/actor/src/test/java/org/thingsboard/server/actors/SlowCreateActor.java
  60. 3
      common/actor/src/test/resources/junit-platform.properties
  61. 0
      common/actor/src/test/resources/logback-test.xml
  62. 5
      common/cache/pom.xml
  63. 21
      common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCacheEvictEvent.java
  64. 42
      common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCacheKey.java
  65. 33
      common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCaffeineCache.java
  66. 35
      common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerRedisCache.java
  67. 11
      common/cache/src/main/java/org/thingsboard/server/cache/limits/DefaultRateLimitService.java
  68. 2
      common/cache/src/main/java/org/thingsboard/server/cache/limits/RateLimitService.java
  69. 21
      common/cache/src/main/java/org/thingsboard/server/cache/user/UserCacheEvictEvent.java
  70. 42
      common/cache/src/main/java/org/thingsboard/server/cache/user/UserCacheKey.java
  71. 33
      common/cache/src/main/java/org/thingsboard/server/cache/user/UserCaffeineCache.java
  72. 35
      common/cache/src/main/java/org/thingsboard/server/cache/user/UserRedisCache.java
  73. 5
      common/cache/src/test/java/org/thingsboard/server/cache/CacheSpecsMapTest.java
  74. 3
      common/cache/src/test/resources/junit-platform.properties
  75. 0
      common/cache/src/test/resources/logback-test.xml
  76. 5
      common/cluster-api/pom.xml
  77. 5
      common/dao-api/pom.xml
  78. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java
  79. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/customer/CustomerService.java
  80. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java
  81. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java
  82. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java
  83. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java
  84. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java
  85. 5
      common/data/pom.xml
  86. 2
      common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java
  87. 4
      common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java
  88. 13
      common/data/src/main/java/org/thingsboard/server/common/data/HasDefaultOption.java
  89. 3
      common/data/src/main/java/org/thingsboard/server/common/data/asset/AssetProfile.java
  90. 1
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/lwm2m/OtherConfiguration.java
  91. 30
      common/data/src/main/java/org/thingsboard/server/common/data/exception/RateLimitExceededException.java
  92. 3
      common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java
  93. 2
      common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java
  94. 12
      common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java
  95. 9
      common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java
  96. 2
      common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtSettings.java
  97. 14
      common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java
  98. 17
      common/data/src/test/java/org/thingsboard/server/common/data/UUIDConverterTest.java
  99. 6
      common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java
  100. 3
      common/data/src/test/resources/junit-platform.properties

5
application/src/main/data/json/system/widget_bundles/charts.json

@ -21,7 +21,7 @@
"doughnut", "doughnut",
"horizontal_doughnut", "horizontal_doughnut",
"polar_area", "polar_area",
"charts.radar_chart_js", "radar",
"charts.basic_timeseries", "charts.basic_timeseries",
"charts.state_chart", "charts.state_chart",
"charts.timeseries_bars_flot", "charts.timeseries_bars_flot",
@ -29,6 +29,7 @@
"charts.pie", "charts.pie",
"charts.pie_chart_js", "charts.pie_chart_js",
"charts.doughnut_chart_js", "charts.doughnut_chart_js",
"charts.polar_area_chart_js" "charts.polar_area_chart_js",
"charts.radar_chart_js"
] ]
} }

2
application/src/main/data/json/system/widget_types/alarms_table.json

@ -11,7 +11,7 @@
"resources": [], "resources": [],
"templateHtml": "<tb-alarms-table-widget \n [ctx]=\"ctx\">\n</tb-alarms-table-widget>", "templateHtml": "<tb-alarms-table-widget \n [ctx]=\"ctx\">\n</tb-alarms-table-widget>",
"templateCss": "", "templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.alarmsTableWidget.onDataUpdated();\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.alarmsTableWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.alarmsTableWidget.onEditModeChanged();\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'cellClick': {\n name: 'widget-action.cell-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "", "settingsSchema": "",
"dataKeySettingsSchema": "", "dataKeySettingsSchema": "",
"settingsDirective": "tb-alarms-table-widget-settings", "settingsDirective": "tb-alarms-table-widget-settings",

2
application/src/main/data/json/system/widget_types/asset_admin_table.json

@ -11,7 +11,7 @@
"resources": [], "resources": [],
"templateHtml": "<tb-entities-table-widget \n [ctx]=\"ctx\">\n</tb-entities-table-widget>", "templateHtml": "<tb-entities-table-widget \n [ctx]=\"ctx\">\n</tb-entities-table-widget>",
"templateCss": "", "templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.entitiesTableWidget.onEditModeChanged();\n}\n\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n },\n 'cellClick': {\n name: 'widget-action.cell-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "", "settingsSchema": "",
"dataKeySettingsSchema": "", "dataKeySettingsSchema": "",
"settingsDirective": "tb-entities-table-widget-settings", "settingsDirective": "tb-entities-table-widget-settings",

2
application/src/main/data/json/system/widget_types/device_admin_table.json

@ -11,7 +11,7 @@
"resources": [], "resources": [],
"templateHtml": "<tb-entities-table-widget \n [ctx]=\"ctx\">\n</tb-entities-table-widget>", "templateHtml": "<tb-entities-table-widget \n [ctx]=\"ctx\">\n</tb-entities-table-widget>",
"templateCss": "", "templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.entitiesTableWidget.onEditModeChanged();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n },\n 'cellClick': {\n name: 'widget-action.cell-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "", "settingsSchema": "",
"dataKeySettingsSchema": "", "dataKeySettingsSchema": "",
"settingsDirective": "tb-entities-table-widget-settings", "settingsDirective": "tb-entities-table-widget-settings",

2
application/src/main/data/json/system/widget_types/entities_hierarchy.json

@ -11,7 +11,7 @@
"resources": [], "resources": [],
"templateHtml": "<tb-entities-hierarchy-widget \n [ctx]=\"ctx\">\n</tb-entities-hierarchy-widget>", "templateHtml": "<tb-entities-hierarchy-widget \n [ctx]=\"ctx\">\n</tb-entities-hierarchy-widget>",
"templateCss": "", "templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesHierarchyWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'nodeSelected': {\n name: 'widget-action.node-selected',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesHierarchyWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.entitiesHierarchyWidget.onEditModeChanged();\n}\n\nself.typeParameters = function() {\n return {\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'nodeSelected': {\n name: 'widget-action.node-selected',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "", "settingsSchema": "",
"dataKeySettingsSchema": "", "dataKeySettingsSchema": "",
"settingsDirective": "tb-entities-hierarchy-widget-settings", "settingsDirective": "tb-entities-hierarchy-widget-settings",

2
application/src/main/data/json/system/widget_types/entities_table.json

@ -11,7 +11,7 @@
"resources": [], "resources": [],
"templateHtml": "<tb-entities-table-widget \n [ctx]=\"ctx\">\n</tb-entities-table-widget>", "templateHtml": "<tb-entities-table-widget \n [ctx]=\"ctx\">\n</tb-entities-table-widget>",
"templateCss": "", "templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'name', type: 'entityField' }];\n }\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.entitiesTableWidget.onEditModeChanged();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'name', type: 'entityField' }];\n }\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n },\n 'cellClick': {\n name: 'widget-action.cell-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "", "settingsSchema": "",
"dataKeySettingsSchema": "", "dataKeySettingsSchema": "",
"settingsDirective": "tb-entities-table-widget-settings", "settingsDirective": "tb-entities-table-widget-settings",

33
application/src/main/data/json/system/widget_types/radar.json

File diff suppressed because one or more lines are too long

25
application/src/main/data/json/system/widget_types/radar_deprecated.json

File diff suppressed because one or more lines are too long

2
application/src/main/data/json/system/widget_types/timeseries_table.json

@ -11,7 +11,7 @@
"resources": [], "resources": [],
"templateHtml": "<tb-timeseries-table-widget \n [ctx]=\"ctx\">\n</tb-timeseries-table-widget>", "templateHtml": "<tb-timeseries-table-widget \n [ctx]=\"ctx\">\n</tb-timeseries-table-widget>",
"templateCss": "", "templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeseriesTableWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeseriesTableWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n ignoreDataUpdateOnIntervalTick: true,\n hasAdditionalLatestDataKeys: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeseriesTableWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeseriesTableWidget.onLatestDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.timeseriesTableWidget.onEditModeChanged();\n}\n\nself.typeParameters = function() {\n return {\n ignoreDataUpdateOnIntervalTick: true,\n hasAdditionalLatestDataKeys: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "", "settingsSchema": "",
"dataKeySettingsSchema": "", "dataKeySettingsSchema": "",
"latestDataKeySettingsSchema": "", "latestDataKeySettingsSchema": "",

17
application/src/main/data/json/tenant/dashboards/gateways.json

File diff suppressed because one or more lines are too long

25
application/src/main/data/upgrade/3.6.4/schema_update.sql

@ -14,6 +14,23 @@
-- limitations under the License. -- limitations under the License.
-- --
-- UPDATE PUBLIC CUSTOMERS START
DO
$$
BEGIN
IF NOT EXISTS (
SELECT FROM information_schema.columns
WHERE table_name = 'customer' AND column_name = 'is_public'
) THEN
ALTER TABLE customer ADD COLUMN is_public boolean DEFAULT false;
UPDATE customer SET is_public = true WHERE title = 'Public';
END IF;
END;
$$;
-- UPDATE PUBLIC CUSTOMERS END
-- create new attribute_kv table schema -- create new attribute_kv table schema
DO DO
$$ $$
@ -90,13 +107,7 @@ BEGIN
SELECT COUNT(*) INTO row_num_old FROM attribute_kv_old; SELECT COUNT(*) INTO row_num_old FROM attribute_kv_old;
SELECT COUNT(*) INTO row_num FROM attribute_kv; SELECT COUNT(*) INTO row_num FROM attribute_kv;
RAISE NOTICE 'Migrated % of % rows', row_num, row_num_old; RAISE NOTICE 'Migrated % of % rows', row_num, row_num_old;
DROP TABLE IF EXISTS attribute_kv_old;
IF row_num != 0 THEN
DROP TABLE IF EXISTS attribute_kv_old;
ELSE
RAISE EXCEPTION 'Table attribute_kv is empty';
END IF;
CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc); CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);
END IF; END IF;
EXCEPTION EXCEPTION

4
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java

@ -103,7 +103,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
RuleChain ruleChain = service.findRuleChainById(tenantId, entityId); RuleChain ruleChain = service.findRuleChainById(tenantId, entityId);
if (ruleChain != null && RuleChainType.CORE.equals(ruleChain.getType())) { if (ruleChain != null && RuleChainType.CORE.equals(ruleChain.getType())) {
List<RuleNode> ruleNodeList = service.getRuleChainNodes(tenantId, entityId); List<RuleNode> ruleNodeList = service.getRuleChainNodes(tenantId, entityId);
log.trace("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); log.debug("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size());
// Creating and starting the actors; // Creating and starting the actors;
for (RuleNode ruleNode : ruleNodeList) { for (RuleNode ruleNode : ruleNodeList) {
log.trace("[{}][{}] Creating rule node [{}]: {}", entityId, ruleNode.getId(), ruleNode.getName(), ruleNode); log.trace("[{}][{}] Creating rule node [{}]: {}", entityId, ruleNode.getId(), ruleNode.getName(), ruleNode);
@ -124,7 +124,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
if (ruleChain != null && RuleChainType.CORE.equals(ruleChain.getType())) { if (ruleChain != null && RuleChainType.CORE.equals(ruleChain.getType())) {
ruleChainName = ruleChain.getName(); ruleChainName = ruleChain.getName();
List<RuleNode> ruleNodeList = service.getRuleChainNodes(tenantId, entityId); List<RuleNode> ruleNodeList = service.getRuleChainNodes(tenantId, entityId);
log.trace("[{}][{}] Updating rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); log.debug("[{}][{}] Updating rule chain with {} nodes", tenantId, entityId, ruleNodeList.size());
for (RuleNode ruleNode : ruleNodeList) { for (RuleNode ruleNode : ruleNodeList) {
RuleNodeCtx existing = nodeActors.get(ruleNode.getId()); RuleNodeCtx existing = nodeActors.get(ruleNode.getId());
if (existing == null) { if (existing == null) {

28
application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java

@ -72,12 +72,11 @@ public class ThingsboardSecurityConfiguration {
public static final String JWT_TOKEN_HEADER_PARAM_V2 = "Authorization"; public static final String JWT_TOKEN_HEADER_PARAM_V2 = "Authorization";
public static final String JWT_TOKEN_QUERY_PARAM = "token"; public static final String JWT_TOKEN_QUERY_PARAM = "token";
public static final String WEBJARS_ENTRY_POINT = "/webjars/**";
public static final String DEVICE_API_ENTRY_POINT = "/api/v1/**"; public static final String DEVICE_API_ENTRY_POINT = "/api/v1/**";
public static final String FORM_BASED_LOGIN_ENTRY_POINT = "/api/auth/login"; public static final String FORM_BASED_LOGIN_ENTRY_POINT = "/api/auth/login";
public static final String PUBLIC_LOGIN_ENTRY_POINT = "/api/auth/login/public"; public static final String PUBLIC_LOGIN_ENTRY_POINT = "/api/auth/login/public";
public static final String TOKEN_REFRESH_ENTRY_POINT = "/api/auth/token"; public static final String TOKEN_REFRESH_ENTRY_POINT = "/api/auth/token";
protected static final String[] NON_TOKEN_BASED_AUTH_ENTRY_POINTS = new String[] {"/index.html", "/assets/**", "/static/**", "/api/noauth/**", "/webjars/**", "/api/license/**", "/api/images/public/**"}; protected static final String[] NON_TOKEN_BASED_AUTH_ENTRY_POINTS = new String[]{"/index.html", "/assets/**", "/static/**", "/api/noauth/**", "/webjars/**", "/api/license/**", "/api/images/public/**"};
public static final String TOKEN_BASED_AUTH_ENTRY_POINT = "/api/**"; public static final String TOKEN_BASED_AUTH_ENTRY_POINT = "/api/**";
public static final String WS_ENTRY_POINT = "/api/ws/**"; public static final String WS_ENTRY_POINT = "/api/ws/**";
public static final String MAIL_OAUTH2_PROCESSING_ENTRY_POINT = "/api/admin/mail/oauth2/code"; public static final String MAIL_OAUTH2_PROCESSING_ENTRY_POINT = "/api/admin/mail/oauth2/code";
@ -153,7 +152,7 @@ public class ThingsboardSecurityConfiguration {
protected JwtTokenAuthenticationProcessingFilter buildJwtTokenAuthenticationProcessingFilter() throws Exception { protected JwtTokenAuthenticationProcessingFilter buildJwtTokenAuthenticationProcessingFilter() throws Exception {
List<String> pathsToSkip = new ArrayList<>(Arrays.asList(NON_TOKEN_BASED_AUTH_ENTRY_POINTS)); List<String> pathsToSkip = new ArrayList<>(Arrays.asList(NON_TOKEN_BASED_AUTH_ENTRY_POINTS));
pathsToSkip.addAll(Arrays.asList(WS_ENTRY_POINT, TOKEN_REFRESH_ENTRY_POINT, FORM_BASED_LOGIN_ENTRY_POINT, pathsToSkip.addAll(Arrays.asList(WS_ENTRY_POINT, TOKEN_REFRESH_ENTRY_POINT, FORM_BASED_LOGIN_ENTRY_POINT,
PUBLIC_LOGIN_ENTRY_POINT, DEVICE_API_ENTRY_POINT, WEBJARS_ENTRY_POINT, MAIL_OAUTH2_PROCESSING_ENTRY_POINT, PUBLIC_LOGIN_ENTRY_POINT, DEVICE_API_ENTRY_POINT, MAIL_OAUTH2_PROCESSING_ENTRY_POINT,
DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT)); DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT));
SkipPathRequestMatcher matcher = new SkipPathRequestMatcher(pathsToSkip, TOKEN_BASED_AUTH_ENTRY_POINT); SkipPathRequestMatcher matcher = new SkipPathRequestMatcher(pathsToSkip, TOKEN_BASED_AUTH_ENTRY_POINT);
JwtTokenAuthenticationProcessingFilter filter JwtTokenAuthenticationProcessingFilter filter
@ -210,18 +209,17 @@ public class ThingsboardSecurityConfiguration {
.exceptionHandling(config -> {}) .exceptionHandling(config -> {})
.sessionManagement(config -> config.sessionCreationPolicy(SessionCreationPolicy.STATELESS)) .sessionManagement(config -> config.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
.authorizeHttpRequests(config -> config .authorizeHttpRequests(config -> config
.requestMatchers(WEBJARS_ENTRY_POINT).permitAll() // Webjars .requestMatchers(NON_TOKEN_BASED_AUTH_ENTRY_POINTS).permitAll() // static resources, user activation and password reset end-points (webjars included)
.requestMatchers(DEVICE_API_ENTRY_POINT).permitAll() // Device HTTP Transport API .requestMatchers(
.requestMatchers(FORM_BASED_LOGIN_ENTRY_POINT).permitAll() // Login end-point DEVICE_API_ENTRY_POINT, // Device HTTP Transport API
.requestMatchers(PUBLIC_LOGIN_ENTRY_POINT).permitAll() // Public login end-point FORM_BASED_LOGIN_ENTRY_POINT, // Login end-point
.requestMatchers(TOKEN_REFRESH_ENTRY_POINT).permitAll() // Token refresh end-point PUBLIC_LOGIN_ENTRY_POINT, // Public login end-point
.requestMatchers(MAIL_OAUTH2_PROCESSING_ENTRY_POINT).permitAll() // Mail oauth2 code processing url TOKEN_REFRESH_ENTRY_POINT, // Token refresh end-point
.requestMatchers(DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT).permitAll() // Device connectivity certificate (public) MAIL_OAUTH2_PROCESSING_ENTRY_POINT, // Mail oauth2 code processing url
.requestMatchers(NON_TOKEN_BASED_AUTH_ENTRY_POINTS).permitAll() // static resources, user activation and password reset end-points DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT, // Device connectivity certificate (public)
.requestMatchers(WS_ENTRY_POINT).permitAll() // Protected WebSocket API End-points WS_ENTRY_POINT).permitAll() // Protected WebSocket API End-points
.requestMatchers(TOKEN_BASED_AUTH_ENTRY_POINT).authenticated()) // Protected API End-points .requestMatchers(TOKEN_BASED_AUTH_ENTRY_POINT).authenticated() // Protected API End-points
.formLogin(form -> form .anyRequest().permitAll())
.loginPage("/login").permitAll())
.exceptionHandling(config -> config.accessDeniedHandler(restAccessDeniedHandler)) .exceptionHandling(config -> config.accessDeniedHandler(restAccessDeniedHandler))
.addFilterBefore(buildRestLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildRestLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(buildRestPublicLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildRestPublicLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class)

3
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -133,7 +133,10 @@ public class ThingsboardInstallService {
case "3.6.4": case "3.6.4":
log.info("Upgrading ThingsBoard from version 3.6.4 to 3.7.0 ..."); log.info("Upgrading ThingsBoard from version 3.6.4 to 3.7.0 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.6.4"); databaseEntitiesUpgradeService.upgradeDatabase("3.6.4");
dataUpdateService.updateData("3.6.4");
entityDatabaseSchemaService.createCustomerTitleUniqueConstraintIfNotExists();
systemDataLoaderService.updateDefaultNotificationConfigs(false); systemDataLoaderService.updateDefaultNotificationConfigs(false);
systemDataLoaderService.updateJwtSettings();
//TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache
break; break;
default: default:

5
application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java

@ -33,7 +33,6 @@ import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MClientCredential; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MClientCredential;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode;
import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration;
import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.device.data.DeviceData;
import org.thingsboard.server.common.data.device.data.PowerMode; import org.thingsboard.server.common.data.device.data.PowerMode;
import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration;
@ -68,6 +67,8 @@ import java.util.Set;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import static org.eclipse.leshan.core.LwM2m.Version.V1_0;
@Service @Service
@TbCoreComponent @TbCoreComponent
@RequiredArgsConstructor @RequiredArgsConstructor
@ -256,7 +257,7 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
Lwm2mDeviceProfileTransportConfiguration transportConfiguration = new Lwm2mDeviceProfileTransportConfiguration(); Lwm2mDeviceProfileTransportConfiguration transportConfiguration = new Lwm2mDeviceProfileTransportConfiguration();
transportConfiguration.setBootstrap(Collections.emptyList()); transportConfiguration.setBootstrap(Collections.emptyList());
transportConfiguration.setClientLwM2mSettings(new OtherConfiguration(1, 1, 1, PowerMode.DRX, null, null, null, null, null)); transportConfiguration.setClientLwM2mSettings(new OtherConfiguration(1, 1, 1, PowerMode.DRX, null, null, null, null, null, V1_0.toString()));
transportConfiguration.setObserveAttr(new TelemetryMappingConfiguration(Collections.emptyMap(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptyMap())); transportConfiguration.setObserveAttr(new TelemetryMappingConfiguration(Collections.emptyMap(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptyMap()));
DeviceProfileData deviceProfileData = new DeviceProfileData(); DeviceProfileData deviceProfileData = new DeviceProfileData();

2
application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java

@ -27,11 +27,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperService
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.housekeeper.HousekeeperConfig; import org.thingsboard.server.queue.housekeeper.HousekeeperConfig;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.queue.consumer.QueueConsumerManager;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;

2
application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java

@ -27,13 +27,13 @@ import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.housekeeper.HousekeeperConfig; import org.thingsboard.server.queue.housekeeper.HousekeeperConfig;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor; import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor;
import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService; import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService;
import org.thingsboard.server.service.queue.consumer.QueueConsumerManager;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
import java.util.List; import java.util.List;

135
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -19,13 +19,17 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import jakarta.annotation.Nullable;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Getter; import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Profile; import org.springframework.context.annotation.Profile;
import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -81,6 +85,7 @@ import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration;
@ -100,14 +105,11 @@ import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService;
import jakarta.annotation.Nullable; import java.nio.charset.StandardCharsets;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import java.util.Arrays; import java.util.Arrays;
import java.util.Base64;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.TreeMap; import java.util.TreeMap;
@ -117,79 +119,51 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import static org.thingsboard.server.common.data.DataConstants.DEFAULT_DEVICE_TYPE; import static org.thingsboard.server.common.data.DataConstants.DEFAULT_DEVICE_TYPE;
import static org.thingsboard.server.service.security.auth.jwt.settings.DefaultJwtSettingsService.isSigningKeyDefault;
import static org.thingsboard.server.service.security.auth.jwt.settings.DefaultJwtSettingsService.validateTokenSigningKeyLength;
@Service @Service
@Profile("install") @Profile("install")
@Slf4j @Slf4j
@RequiredArgsConstructor
public class DefaultSystemDataLoaderService implements SystemDataLoaderService { public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
public static final String CUSTOMER_CRED = "customer"; public static final String CUSTOMER_CRED = "customer";
public static final String ACTIVITY_STATE = "active"; public static final String ACTIVITY_STATE = "active";
@Autowired private final InstallScripts installScripts;
private InstallScripts installScripts; private final UserService userService;
private final AdminSettingsService adminSettingsService;
private final TenantService tenantService;
private final TenantProfileService tenantProfileService;
private final CustomerService customerService;
private final DeviceService deviceService;
private final DeviceProfileService deviceProfileService;
private final AttributesService attributesService;
private final DeviceCredentialsService deviceCredentialsService;
private final RuleChainService ruleChainService;
private final TimeseriesService tsService;
private final DeviceConnectivityConfiguration connectivityConfiguration;
private final QueueService queueService;
private final JwtSettingsService jwtSettingsService;
private final NotificationSettingsService notificationSettingsService;
private final NotificationTargetService notificationTargetService;
@Autowired @Autowired
private BCryptPasswordEncoder passwordEncoder; private BCryptPasswordEncoder passwordEncoder;
@Autowired
private UserService userService;
@Autowired
private AdminSettingsService adminSettingsService;
@Autowired
private WidgetTypeService widgetTypeService;
@Autowired
private WidgetsBundleService widgetsBundleService;
@Autowired
private TenantService tenantService;
@Autowired
private TenantProfileService tenantProfileService;
@Autowired
private CustomerService customerService;
@Autowired
private DeviceService deviceService;
@Autowired
private DeviceProfileService deviceProfileService;
@Autowired
private AttributesService attributesService;
@Autowired
private DeviceCredentialsService deviceCredentialsService;
@Autowired
private RuleChainService ruleChainService;
@Autowired
private TimeseriesService tsService;
@Autowired
private DeviceConnectivityConfiguration connectivityConfiguration;
@Value("${state.persistToTelemetry:false}") @Value("${state.persistToTelemetry:false}")
@Getter @Getter
private boolean persistActivityToTelemetry; private boolean persistActivityToTelemetry;
@Lazy @Value("${security.jwt.tokenExpirationTime:9000}")
@Autowired private Integer tokenExpirationTime;
private QueueService queueService; @Value("${security.jwt.refreshTokenExpTime:604800}")
private Integer refreshTokenExpTime;
@Autowired @Value("${security.jwt.tokenIssuer:thingsboard.io}")
private JwtSettingsService jwtSettingsService; private String tokenIssuer;
@Value("${security.jwt.tokenSigningKey:thingsboardDefaultSigningKey}")
@Autowired private String tokenSigningKey;
private NotificationSettingsService notificationSettingsService;
@Autowired
private NotificationTargetService notificationTargetService;
@Bean @Bean
protected BCryptPasswordEncoder passwordEncoder() { protected BCryptPasswordEncoder passwordEncoder() {
@ -295,7 +269,42 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
@Override @Override
public void createRandomJwtSettings() throws Exception { public void createRandomJwtSettings() throws Exception {
jwtSettingsService.createRandomJwtSettings(); if (jwtSettingsService.getJwtSettings() == null) {
log.info("Creating JWT admin settings...");
var jwtSettings = new JwtSettings(this.tokenExpirationTime, this.refreshTokenExpTime, this.tokenIssuer, this.tokenSigningKey);
if (isSigningKeyDefault(jwtSettings) || !validateTokenSigningKeyLength(jwtSettings)) {
jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(
RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8)));
}
jwtSettingsService.saveJwtSettings(jwtSettings);
} else {
log.info("Skip creating JWT admin settings because they already exist.");
}
}
@Override
public void updateJwtSettings() {
JwtSettings jwtSettings = jwtSettingsService.getJwtSettings();
boolean invalidSignKey = false;
String warningMessage = null;
if (isSigningKeyDefault(jwtSettings)) {
warningMessage = "The platform is using the default JWT Signing Key, which is a security risk.";
invalidSignKey = true;
} else if (!validateTokenSigningKeyLength(jwtSettings)) {
warningMessage = "The JWT Signing Key is shorter than 512 bits, which is a security risk.";
invalidSignKey = true;
}
if (invalidSignKey) {
log.warn("WARNING: {}. A new JWT Signing Key has been added automatically. " +
"You can change the JWT Signing Key using the Web UI: " +
"Navigate to \"System settings -> Security settings\" while logged in as a System Administrator.", warningMessage);
jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(
RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8)));
jwtSettingsService.saveJwtSettings(jwtSettings);
}
} }
@Override @Override

2
application/src/main/java/org/thingsboard/server/service/install/EntityDatabaseSchemaService.java

@ -21,4 +21,6 @@ public interface EntityDatabaseSchemaService extends DatabaseSchemaService {
void createOrUpdateViewsAndFunctions() throws Exception; void createOrUpdateViewsAndFunctions() throws Exception;
void createCustomerTitleUniqueConstraintIfNotExists();
} }

10
application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java

@ -84,13 +84,17 @@ public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchema
} }
protected void executeQuery(String query) { protected void executeQuery(String query) {
executeQuery(query, null);
}
protected void executeQuery(String query, String logQuery) {
logQuery = logQuery != null ? logQuery : query;
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
log.info("Successfully executed query: {}", query); log.info("Successfully executed query: {}", logQuery);
Thread.sleep(5000); Thread.sleep(5000);
} catch (InterruptedException | SQLException e) { } catch (InterruptedException | SQLException e) {
log.error("Failed to execute query: {} due to: {}", query, e.getMessage()); throw new RuntimeException("Failed to execute query: " + logQuery, e);
throw new RuntimeException("Failed to execute query: " + query, e);
} }
} }

7
application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java

@ -52,4 +52,11 @@ public class SqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaSer
log.info("Installing SQL DataBase schema views and functions: " + SCHEMA_VIEWS_AND_FUNCTIONS_SQL); log.info("Installing SQL DataBase schema views and functions: " + SCHEMA_VIEWS_AND_FUNCTIONS_SQL);
executeQueryFromFile(SCHEMA_VIEWS_AND_FUNCTIONS_SQL); executeQueryFromFile(SCHEMA_VIEWS_AND_FUNCTIONS_SQL);
} }
@Override
public void createCustomerTitleUniqueConstraintIfNotExists() {
executeQuery("DO $$ BEGIN IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'customer_title_unq_key') THEN " +
"ALTER TABLE customer ADD CONSTRAINT customer_title_unq_key UNIQUE(tenant_id, title); END IF; END; $$;",
"create 'customer_title_unq_key' constraint if it doesn't already exist!");
}
} }

2
application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java

@ -25,6 +25,8 @@ public interface SystemDataLoaderService {
void createRandomJwtSettings() throws Exception; void createRandomJwtSettings() throws Exception;
void updateJwtSettings() throws Exception;
void createOAuth2Templates() throws Exception; void createOAuth2Templates() throws Exception;
void loadSystemWidgets() throws Exception; void loadSystemWidgets() throws Exception;

1
application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java

@ -27,7 +27,6 @@ import org.springframework.stereotype.Service;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import static org.thingsboard.server.common.data.CacheConstants.ATTRIBUTES_CACHE;
import static org.thingsboard.server.common.data.CacheConstants.RESOURCE_INFO_CACHE; import static org.thingsboard.server.common.data.CacheConstants.RESOURCE_INFO_CACHE;
import static org.thingsboard.server.common.data.CacheConstants.SECURITY_SETTINGS_CACHE; import static org.thingsboard.server.common.data.CacheConstants.SECURITY_SETTINGS_CACHE;

82
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@ -26,22 +26,27 @@ import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.DynamicValue;
import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.FilterPredicateValue;
import org.thingsboard.server.dao.customer.CustomerDao;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.device.DeviceConnectivityConfiguration; import org.thingsboard.server.dao.device.DeviceConnectivityConfiguration;
import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.sql.JpaExecutorService; import org.thingsboard.server.dao.sql.JpaExecutorService;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.component.RuleNodeClassInfo; import org.thingsboard.server.service.component.RuleNodeClassInfo;
import org.thingsboard.server.utils.TbNodeUpgradeUtils; import org.thingsboard.server.utils.TbNodeUpgradeUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
@Service @Service
@ -67,6 +72,15 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired @Autowired
DeviceConnectivityConfiguration connectivityConfiguration; DeviceConnectivityConfiguration connectivityConfiguration;
@Autowired
private CustomerDao customerDao;
@Autowired
private CustomerService customerService;
@Autowired
private TenantProfileService tenantProfileService;
@Override @Override
public void updateData(String fromVersion) throws Exception { public void updateData(String fromVersion) throws Exception {
switch (fromVersion) { switch (fromVersion) {
@ -74,11 +88,79 @@ public class DefaultDataUpdateService implements DataUpdateService {
log.info("Updating data from version 3.6.0 to 3.6.1 ..."); log.info("Updating data from version 3.6.0 to 3.6.1 ...");
migrateDeviceConnectivity(); migrateDeviceConnectivity();
break; break;
case "3.6.4":
log.info("Updating data from version 3.6.4 to 3.7.0 ...");
updateCustomersWithTheSameTitle();
updateMaxRuleNodeExecsPerMessage();
break;
default: default:
throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion);
} }
} }
private void updateMaxRuleNodeExecsPerMessage() {
var tenantProfiles = new PageDataIterable<>(
link -> tenantProfileService.findTenantProfiles(TenantId.SYS_TENANT_ID, link), DEFAULT_PAGE_SIZE);
tenantProfiles.forEach(tenantProfile -> {
var configurationOpt = tenantProfile.getProfileConfiguration();
configurationOpt.ifPresent(configuration -> {
if (configuration.getMaxRuleNodeExecsPerMessage() == 0) {
configuration.setMaxRuleNodeExecutionsPerMessage(1000);
try {
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile);
} catch (Exception e) {
log.error("Failed to update tenant profile with id: {} due to: ", tenantProfile.getId(), e);
}
}
});
});
}
private void updateCustomersWithTheSameTitle() {
var customers = new ArrayList<Customer>();
new PageDataIterable<>(pageLink ->
customerDao.findCustomersWithTheSameTitle(pageLink), DEFAULT_PAGE_SIZE
).forEach(customers::add);
if (customers.isEmpty()) {
return;
}
var firstCustomer = customers.get(0);
var titleToDeduplicate = firstCustomer.getTitle();
var tenantIdToDeduplicate = firstCustomer.getTenantId();
int duplicateCounter = 1;
for (int i = 1; i < customers.size(); i++) {
var currentCustomer = customers.get(i);
if (currentCustomer.getTitle().equals(titleToDeduplicate) && currentCustomer.getTenantId().equals(tenantIdToDeduplicate)) {
duplicateCounter++;
String currentTitle = currentCustomer.getTitle();
String newTitle = currentTitle + " " + duplicateCounter;
try {
Optional<Customer> customerOpt = customerService.findCustomerByTenantIdAndTitle(tenantIdToDeduplicate, newTitle);
if (customerOpt.isPresent()) {
// fallback logic: customer with title 'currentTitle + " " + duplicateCounter;' might be another duplicate.
newTitle = currentTitle + "_" + currentCustomer.getId();
}
} catch (Exception e) {
log.trace("Failed to find customer with title due to: ", e);
// fallback logic: customer with title 'currentTitle + " " + duplicateCounter;' might be another duplicate.
newTitle = currentTitle + "_" + currentCustomer.getId();
}
currentCustomer.setTitle(newTitle);
try {
customerService.saveCustomer(currentCustomer);
} catch (Exception e) {
log.error("[{}] Failed to update customer with id and title: {}, oldTitle: {}, due to: ",
currentCustomer.getTenantId(), newTitle, currentTitle, e);
}
continue;
}
titleToDeduplicate = currentCustomer.getTitle();
tenantIdToDeduplicate = currentCustomer.getTenantId();
duplicateCounter = 1;
}
}
private void migrateDeviceConnectivity() { private void migrateDeviceConnectivity() {
if (adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "connectivity") == null) { if (adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "connectivity") == null) {
AdminSettings connectivitySettings = new AdminSettings(); AdminSettings connectivitySettings = new AdminSettings();

43
application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java

@ -16,12 +16,13 @@
package org.thingsboard.server.service.mail; package org.thingsboard.server.service.mail;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.util.concurrent.Futures;
import freemarker.template.Configuration; import freemarker.template.Configuration;
import freemarker.template.Template; import freemarker.template.Template;
import jakarta.xml.bind.DatatypeConverter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.exception.ExceptionUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.MessageSource; import org.springframework.context.MessageSource;
import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Lazy;
import org.springframework.core.NestedRuntimeException; import org.springframework.core.NestedRuntimeException;
@ -31,29 +32,36 @@ import org.springframework.mail.javamail.JavaMailSenderImpl;
import org.springframework.mail.javamail.MimeMessageHelper; import org.springframework.mail.javamail.MimeMessageHelper;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.ui.freemarker.FreeMarkerTemplateUtils; import org.springframework.ui.freemarker.FreeMarkerTemplateUtils;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.TbEmail; import org.thingsboard.rule.engine.api.TbEmail;
import org.thingsboard.server.cache.limits.RateLimitService;
import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.ApiFeature; import org.thingsboard.server.common.data.ApiFeature;
import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.ApiUsageRecordState; import org.thingsboard.server.common.data.ApiUsageRecordState;
import org.thingsboard.server.common.data.ApiUsageStateValue; import org.thingsboard.server.common.data.ApiUsageStateValue;
import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.exception.RateLimitExceededException;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.limit.LimitedApi;
import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.common.stats.TbApiUsageReportClient;
import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import jakarta.mail.internet.MimeMessage; import jakarta.mail.internet.MimeMessage;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.util.HashMap; import java.util.HashMap;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
@ -76,13 +84,21 @@ public class DefaultMailService implements MailService {
private TbApiUsageStateService apiUsageStateService; private TbApiUsageStateService apiUsageStateService;
@Autowired @Autowired
private MailExecutorService mailExecutorService; private MailSenderInternalExecutorService mailExecutorService;
@Autowired @Autowired
private PasswordResetExecutorService passwordResetExecutorService; private PasswordResetExecutorService passwordResetExecutorService;
@Autowired @Autowired
private TbMailContextComponent tbMailContextComponent; private TbMailContextComponent ctx;
@Autowired
private RateLimitService rateLimitService;
@Value("${mail.per_tenant_rate_limits:}")
private String perTenantRateLimitConfig;
private final ScheduledExecutorService timeoutScheduler;
private TbMailSender mailSender; private TbMailSender mailSender;
@ -95,6 +111,7 @@ public class DefaultMailService implements MailService {
this.freemarkerConfig = freemarkerConfig; this.freemarkerConfig = freemarkerConfig;
this.adminSettingsService = adminSettingsService; this.adminSettingsService = adminSettingsService;
this.apiUsageClient = apiUsageClient; this.apiUsageClient = apiUsageClient;
this.timeoutScheduler = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("mail-service-watchdog"));
} }
@PostConstruct @PostConstruct
@ -102,12 +119,19 @@ public class DefaultMailService implements MailService {
updateMailConfiguration(); updateMailConfiguration();
} }
@PreDestroy
public void destroy() {
if (timeoutScheduler != null) {
timeoutScheduler.shutdownNow();
}
}
@Override @Override
public void updateMailConfiguration() { public void updateMailConfiguration() {
AdminSettings settings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "mail"); AdminSettings settings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "mail");
if (settings != null) { if (settings != null) {
JsonNode jsonConfig = settings.getJsonValue(); JsonNode jsonConfig = settings.getJsonValue();
mailSender = new TbMailSender(tbMailContextComponent, jsonConfig); mailSender = new TbMailSender(ctx, jsonConfig);
mailFrom = jsonConfig.get("mailFrom").asText(); mailFrom = jsonConfig.get("mailFrom").asText();
timeout = jsonConfig.get("timeout").asLong(DEFAULT_TIMEOUT); timeout = jsonConfig.get("timeout").asLong(DEFAULT_TIMEOUT);
} else { } else {
@ -122,7 +146,7 @@ public class DefaultMailService implements MailService {
@Override @Override
public void sendTestMail(JsonNode jsonConfig, String email) throws ThingsboardException { public void sendTestMail(JsonNode jsonConfig, String email) throws ThingsboardException {
TbMailSender testMailSender = new TbMailSender(tbMailContextComponent, jsonConfig); TbMailSender testMailSender = new TbMailSender(ctx, jsonConfig);
String mailFrom = jsonConfig.get("mailFrom").asText(); String mailFrom = jsonConfig.get("mailFrom").asText();
String subject = messages.getMessage("test.message.subject", null, Locale.US); String subject = messages.getMessage("test.message.subject", null, Locale.US);
long timeout = jsonConfig.get("timeout").asLong(DEFAULT_TIMEOUT); long timeout = jsonConfig.get("timeout").asLong(DEFAULT_TIMEOUT);
@ -214,6 +238,10 @@ public class DefaultMailService implements MailService {
private void sendMail(TenantId tenantId, CustomerId customerId, TbEmail tbEmail, JavaMailSender javaMailSender, long timeout) throws ThingsboardException { private void sendMail(TenantId tenantId, CustomerId customerId, TbEmail tbEmail, JavaMailSender javaMailSender, long timeout) throws ThingsboardException {
if (apiUsageStateService.getApiUsageState(tenantId).isEmailSendEnabled()) { if (apiUsageStateService.getApiUsageState(tenantId).isEmailSendEnabled()) {
if (tenantId != null && !tenantId.isSysTenantId() && StringUtils.isNotEmpty(perTenantRateLimitConfig) &&
!rateLimitService.checkRateLimit(LimitedApi.EMAILS, (Object) tenantId, perTenantRateLimitConfig)) {
throw new RateLimitExceededException(LimitedApi.EMAILS);
}
try { try {
MimeMessage mailMsg = javaMailSender.createMimeMessage(); MimeMessage mailMsg = javaMailSender.createMimeMessage();
boolean multipart = (tbEmail.getImages() != null && !tbEmail.getImages().isEmpty()); boolean multipart = (tbEmail.getImages() != null && !tbEmail.getImages().isEmpty());
@ -415,8 +443,11 @@ public class DefaultMailService implements MailService {
} }
private void sendMailWithTimeout(JavaMailSender mailSender, MimeMessage msg, long timeout) { private void sendMailWithTimeout(JavaMailSender mailSender, MimeMessage msg, long timeout) {
var submittedMail = Futures.withTimeout(
mailExecutorService.submit(() -> mailSender.send(msg)),
timeout, TimeUnit.MILLISECONDS, timeoutScheduler);
try { try {
mailExecutorService.submit(() -> mailSender.send(msg)).get(timeout, TimeUnit.MILLISECONDS); submittedMail.get(timeout, TimeUnit.MILLISECONDS);
} catch (TimeoutException e) { } catch (TimeoutException e) {
log.debug("Error during mail submission", e); log.debug("Error during mail submission", e);
throw new RuntimeException("Timeout!"); throw new RuntimeException("Timeout!");

37
application/src/main/java/org/thingsboard/server/service/mail/MailSenderInternalExecutorService.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.mail;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.common.util.AbstractListeningExecutor;
/**
* Executor have the sole purpose to send mails. It should be used only by Mail Service.
* For other purposes please use the MailExecutorService component
* */
@Component
public class MailSenderInternalExecutorService extends AbstractListeningExecutor {
@Value("${actors.rule.mail_thread_pool_size}")
private int mailExecutorThreadPoolSize;
@Override
protected int getThreadPollSize() {
return mailExecutorThreadPoolSize;
}
}

1
application/src/main/java/org/thingsboard/server/service/mail/TbMailContextComponent.java

@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
@Component @Component
@Data @Data

3
application/src/main/java/org/thingsboard/server/service/notification/channels/MobileAppNotificationChannel.java

@ -105,9 +105,10 @@ public class MobileAppNotificationChannel implements NotificationChannel<User, M
firebaseService.sendMessage(ctx.getTenantId(), credentials, token, subject, body, data, unreadCount); firebaseService.sendMessage(ctx.getTenantId(), credentials, token, subject, body, data, unreadCount);
} catch (FirebaseMessagingException e) { } catch (FirebaseMessagingException e) {
MessagingErrorCode errorCode = e.getMessagingErrorCode(); MessagingErrorCode errorCode = e.getMessagingErrorCode();
if (errorCode == MessagingErrorCode.UNREGISTERED || errorCode == MessagingErrorCode.INVALID_ARGUMENT) { if (errorCode == MessagingErrorCode.UNREGISTERED || errorCode == MessagingErrorCode.INVALID_ARGUMENT || errorCode == MessagingErrorCode.SENDER_ID_MISMATCH) {
validTokens.remove(token); validTokens.remove(token);
userService.removeMobileSession(recipient.getTenantId(), token); userService.removeMobileSession(recipient.getTenantId(), token);
log.debug("[{}][{}] Removed invalid FCM token due to {} {} ({})", recipient.getTenantId(), recipient.getId(), errorCode, e.getMessage(), token);
continue; continue;
} }
throw new RuntimeException("Failed to send message via FCM: " + e.getMessage(), e); throw new RuntimeException("Failed to send message via FCM: " + e.getMessage(), e);

416
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -20,12 +20,12 @@ import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy; import jakarta.annotation.PreDestroy;
import lombok.Data;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.ApplicationEventPublisher;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.id.NotificationRequestId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTrigger; import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTrigger;
import org.thingsboard.server.common.data.queue.QueueConfig;
import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.data.rpc.RpcError;
import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbActorMsg;
@ -78,10 +79,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM
import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.edge.EdgeNotificationService; import org.thingsboard.server.service.edge.EdgeNotificationService;
@ -89,6 +91,7 @@ import org.thingsboard.server.service.notification.NotificationSchedulerService;
import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.ota.OtaPackageStateService;
import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.queue.processing.IdMsgPair;
import org.thingsboard.server.service.resource.TbImageService; import org.thingsboard.server.service.resource.TbImageService;
@ -109,7 +112,6 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -122,9 +124,11 @@ import java.util.stream.Collectors;
public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCoreNotificationMsg> implements TbCoreConsumerService { public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCoreNotificationMsg> implements TbCoreConsumerService {
@Value("${queue.core.poll-interval}") @Value("${queue.core.poll-interval}")
private long pollDuration; private long pollInterval;
@Value("${queue.core.pack-processing-timeout}") @Value("${queue.core.pack-processing-timeout}")
private long packProcessingTimeout; private long packProcessingTimeout;
@Value("${queue.core.consumer-per-partition:true}")
private boolean consumerPerPartition;
@Value("${queue.core.stats.enabled:false}") @Value("${queue.core.stats.enabled:false}")
private boolean statsEnabled; private boolean statsEnabled;
@ -133,7 +137,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
@Value("${queue.core.ota.pack-size:100}") @Value("${queue.core.ota.pack-size:100}")
private int firmwarePackSize; private int firmwarePackSize;
private final TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> mainConsumer;
private final DeviceStateService stateService; private final DeviceStateService stateService;
private final TbApiUsageStateService statsService; private final TbApiUsageStateService statsService;
private final TbLocalSubscriptionService localSubscriptionService; private final TbLocalSubscriptionService localSubscriptionService;
@ -144,14 +147,14 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
private final GitVersionControlQueueService vcQueueService; private final GitVersionControlQueueService vcQueueService;
private final NotificationSchedulerService notificationSchedulerService; private final NotificationSchedulerService notificationSchedulerService;
private final NotificationRuleProcessor notificationRuleProcessor; private final NotificationRuleProcessor notificationRuleProcessor;
private final TbCoreConsumerStats stats; private final TbCoreQueueFactory queueFactory;
protected final TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
private final TbQueueConsumer<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> firmwareStatesConsumer;
private final TbImageService imageService; private final TbImageService imageService;
private final TbCoreConsumerStats stats;
private MainQueueConsumerManager<TbProtoQueueMsg<ToCoreMsg>, CoreQueueConfig> mainConsumer;
private QueueConsumerManager<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
private QueueConsumerManager<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> firmwareStatesConsumer;
protected volatile ExecutorService consumersExecutor;
protected volatile ExecutorService usageStatsExecutor;
private volatile ExecutorService firmwareStatesExecutor;
private volatile ListeningExecutorService deviceActivityEventsExecutor; private volatile ListeningExecutorService deviceActivityEventsExecutor;
public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory,
@ -175,10 +178,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
NotificationSchedulerService notificationSchedulerService, NotificationSchedulerService notificationSchedulerService,
NotificationRuleProcessor notificationRuleProcessor, NotificationRuleProcessor notificationRuleProcessor,
TbImageService imageService) { TbImageService imageService) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer(), jwtSettingsService); super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService,
this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); eventPublisher, jwtSettingsService);
this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer();
this.firmwareStatesConsumer = tbCoreQueueFactory.createToOtaPackageStateServiceMsgConsumer();
this.stateService = stateService; this.stateService = stateService;
this.localSubscriptionService = localSubscriptionService; this.localSubscriptionService = localSubscriptionService;
this.subscriptionManagerService = subscriptionManagerService; this.subscriptionManagerService = subscriptionManagerService;
@ -191,152 +192,146 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
this.notificationSchedulerService = notificationSchedulerService; this.notificationSchedulerService = notificationSchedulerService;
this.notificationRuleProcessor = notificationRuleProcessor; this.notificationRuleProcessor = notificationRuleProcessor;
this.imageService = imageService; this.imageService = imageService;
this.queueFactory = tbCoreQueueFactory;
} }
@PostConstruct @PostConstruct
public void init() { public void init() {
super.init("tb-core-notifications-consumer"); super.init("tb-core");
this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-consumer"));
this.usageStatsExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer"));
this.firmwareStatesExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-firmware-notifications-consumer"));
this.deviceActivityEventsExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-device-activity-events-executor"))); this.deviceActivityEventsExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-device-activity-events-executor")));
this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToCoreMsg>, CoreQueueConfig>builder()
.queueKey(new QueueKey(ServiceType.TB_CORE))
.config(CoreQueueConfig.of(consumerPerPartition, (int) pollInterval))
.msgPackProcessor(this::processMsgs)
.consumerCreator(config -> queueFactory.createToCoreMsgConsumer())
.consumerExecutor(consumersExecutor)
.scheduler(scheduler)
.taskExecutor(mgmtExecutor)
.build();
this.usageStatsConsumer = QueueConsumerManager.<TbProtoQueueMsg<ToUsageStatsServiceMsg>>builder()
.name("TB Usage Stats")
.msgPackProcessor(this::processUsageStatsMsg)
.pollInterval(pollInterval)
.consumerCreator(queueFactory::createToUsageStatsServiceMsgConsumer)
.consumerExecutor(consumersExecutor)
.threadPrefix("usage-stats")
.build();
this.firmwareStatesConsumer = QueueConsumerManager.<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>>builder()
.name("TB Ota Package States")
.msgPackProcessor(this::processFirmwareMsgs)
.pollInterval(pollInterval)
.consumerCreator(queueFactory::createToOtaPackageStateServiceMsgConsumer)
.consumerExecutor(consumersExecutor)
.threadPrefix("firmware")
.build();
} }
@PreDestroy @PreDestroy
public void destroy() { public void destroy() {
super.destroy(); super.destroy();
if (consumersExecutor != null) {
consumersExecutor.shutdownNow();
}
if (usageStatsExecutor != null) {
usageStatsExecutor.shutdownNow();
}
if (firmwareStatesExecutor != null) {
firmwareStatesExecutor.shutdownNow();
}
if (deviceActivityEventsExecutor != null) { if (deviceActivityEventsExecutor != null) {
deviceActivityEventsExecutor.shutdownNow(); deviceActivityEventsExecutor.shutdownNow();
} }
} }
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) @Override
public void onApplicationEvent(ApplicationReadyEvent event) { protected void startConsumers() {
super.onApplicationEvent(event); super.startConsumers();
launchUsageStatsConsumer(); firmwareStatesConsumer.subscribe();
launchOtaPackageUpdateNotificationConsumer(); firmwareStatesConsumer.launch();
usageStatsConsumer.launch();
} }
@Override @Override
protected void onTbApplicationEvent(PartitionChangeEvent event) { protected void onTbApplicationEvent(PartitionChangeEvent event) {
log.info("Subscribing to partitions: {}", event.getPartitions()); log.info("Subscribing to partitions: {}", event.getPartitions());
this.mainConsumer.subscribe(event.getPartitions()); mainConsumer.update(event.getPartitions());
this.usageStatsConsumer.subscribe( usageStatsConsumer.subscribe(event.getPartitions()
event .stream()
.getPartitions() .map(tpi -> tpi.newByTopic(usageStatsConsumer.getConsumer().getTopic()))
.stream() .collect(Collectors.toSet()));
.map(tpi -> tpi.newByTopic(usageStatsConsumer.getTopic())) }
.collect(Collectors.toSet()));
this.firmwareStatesConsumer.subscribe(); private void processMsgs(List<TbProtoQueueMsg<ToCoreMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> consumer, CoreQueueConfig config) throws Exception {
} List<IdMsgPair<ToCoreMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).collect(Collectors.toList());
ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = orderedMsgList.stream().collect(
@Override Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
protected void launchMainConsumers() { CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
consumersExecutor.submit(() -> { TbPackProcessingContext<TbProtoQueueMsg<ToCoreMsg>> ctx = new TbPackProcessingContext<>(
while (!stopped) { processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
PendingMsgHolder pendingMsgHolder = new PendingMsgHolder();
Future<?> packSubmitFuture = consumersExecutor.submit(() -> {
orderedMsgList.forEach((element) -> {
UUID id = element.getUuid();
TbProtoQueueMsg<ToCoreMsg> msg = element.getMsg();
log.trace("[{}] Creating main callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try { try {
List<TbProtoQueueMsg<ToCoreMsg>> msgs = mainConsumer.poll(pollDuration); ToCoreMsg toCoreMsg = msg.getValue();
if (msgs.isEmpty()) { pendingMsgHolder.setToCoreMsg(toCoreMsg);
continue; if (toCoreMsg.hasToSubscriptionMgrMsg()) {
} log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg());
List<IdMsgPair<ToCoreMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).collect(Collectors.toList()); forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback);
ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = orderedMsgList.stream().collect( } else if (toCoreMsg.hasToDeviceActorMsg()) {
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg());
CountDownLatch processingTimeoutLatch = new CountDownLatch(1); forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback);
TbPackProcessingContext<TbProtoQueueMsg<ToCoreMsg>> ctx = new TbPackProcessingContext<>( } else if (toCoreMsg.hasDeviceStateServiceMsg()) {
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceStateServiceMsg());
PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback);
Future<?> packSubmitFuture = consumersExecutor.submit(() -> { } else if (toCoreMsg.hasEdgeNotificationMsg()) {
orderedMsgList.forEach((element) -> { log.trace("[{}] Forwarding message to edge service {}", id, toCoreMsg.getEdgeNotificationMsg());
UUID id = element.getUuid(); forwardToEdgeNotificationService(toCoreMsg.getEdgeNotificationMsg(), callback);
TbProtoQueueMsg<ToCoreMsg> msg = element.getMsg(); } else if (toCoreMsg.hasDeviceConnectMsg()) {
log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceConnectMsg());
TbCallback callback = new TbPackCallback<>(id, ctx); forwardToStateService(toCoreMsg.getDeviceConnectMsg(), callback);
try { } else if (toCoreMsg.hasDeviceActivityMsg()) {
ToCoreMsg toCoreMsg = msg.getValue(); log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceActivityMsg());
pendingMsgHolder.setToCoreMsg(toCoreMsg); forwardToStateService(toCoreMsg.getDeviceActivityMsg(), callback);
if (toCoreMsg.hasToSubscriptionMgrMsg()) { } else if (toCoreMsg.hasDeviceDisconnectMsg()) {
log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceDisconnectMsg());
forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); forwardToStateService(toCoreMsg.getDeviceDisconnectMsg(), callback);
} else if (toCoreMsg.hasToDeviceActorMsg()) { } else if (toCoreMsg.hasDeviceInactivityMsg()) {
log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceInactivityMsg());
forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); forwardToStateService(toCoreMsg.getDeviceInactivityMsg(), callback);
} else if (toCoreMsg.hasDeviceStateServiceMsg()) { } else if (toCoreMsg.hasToDeviceActorNotification()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); TbActorMsg actorMsg = ProtoUtils.fromProto(toCoreMsg.getToDeviceActorNotification());
forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); if (actorMsg != null) {
} else if (toCoreMsg.hasEdgeNotificationMsg()) { if (actorMsg.getMsgType().equals(MsgType.DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG)) {
log.trace("[{}] Forwarding message to edge service {}", id, toCoreMsg.getEdgeNotificationMsg()); tbCoreDeviceRpcService.forwardRpcRequestToDeviceActor((ToDeviceRpcRequestActorMsg) actorMsg);
forwardToEdgeNotificationService(toCoreMsg.getEdgeNotificationMsg(), callback); } else {
} else if (toCoreMsg.hasDeviceConnectMsg()) { log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg);
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceConnectMsg()); actorContext.tell(actorMsg);
forwardToStateService(toCoreMsg.getDeviceConnectMsg(), callback);
} else if (toCoreMsg.hasDeviceActivityMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceActivityMsg());
forwardToStateService(toCoreMsg.getDeviceActivityMsg(), callback);
} else if (toCoreMsg.hasDeviceDisconnectMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceDisconnectMsg());
forwardToStateService(toCoreMsg.getDeviceDisconnectMsg(), callback);
} else if (toCoreMsg.hasDeviceInactivityMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceInactivityMsg());
forwardToStateService(toCoreMsg.getDeviceInactivityMsg(), callback);
} else if (toCoreMsg.hasToDeviceActorNotification()) {
TbActorMsg actorMsg = ProtoUtils.fromProto(toCoreMsg.getToDeviceActorNotification());
if (actorMsg != null) {
if (actorMsg.getMsgType().equals(MsgType.DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG)) {
tbCoreDeviceRpcService.forwardRpcRequestToDeviceActor((ToDeviceRpcRequestActorMsg) actorMsg);
} else {
log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg);
actorContext.tell(actorMsg);
}
}
callback.onSuccess();
} else if (toCoreMsg.hasNotificationSchedulerServiceMsg()) {
TransportProtos.NotificationSchedulerServiceMsg notificationSchedulerServiceMsg = toCoreMsg.getNotificationSchedulerServiceMsg();
log.trace("[{}] Forwarding message to notification scheduler service {}", id, toCoreMsg.getNotificationSchedulerServiceMsg());
forwardToNotificationSchedulerService(notificationSchedulerServiceMsg, callback);
} else if (toCoreMsg.hasErrorEventMsg()) {
forwardToEventService(toCoreMsg.getErrorEventMsg(), callback);
} else if (toCoreMsg.hasLifecycleEventMsg()) {
forwardToEventService(toCoreMsg.getLifecycleEventMsg(), callback);
}
} catch (Throwable e) {
log.warn("[{}] Failed to process message: {}", id, msg, e);
callback.onFailure(e);
} }
});
});
if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) {
if (!packSubmitFuture.isDone()) {
packSubmitFuture.cancel(true);
ToCoreMsg lastSubmitMsg = pendingMsgHolder.getToCoreMsg();
log.info("Timeout to process message: {}", lastSubmitMsg);
}
ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue()));
}
mainConsumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to obtain messages from queue.", e);
try {
Thread.sleep(pollDuration);
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new requests", e2);
} }
callback.onSuccess();
} else if (toCoreMsg.hasNotificationSchedulerServiceMsg()) {
TransportProtos.NotificationSchedulerServiceMsg notificationSchedulerServiceMsg = toCoreMsg.getNotificationSchedulerServiceMsg();
log.trace("[{}] Forwarding message to notification scheduler service {}", id, toCoreMsg.getNotificationSchedulerServiceMsg());
forwardToNotificationSchedulerService(notificationSchedulerServiceMsg, callback);
} else if (toCoreMsg.hasErrorEventMsg()) {
forwardToEventService(toCoreMsg.getErrorEventMsg(), callback);
} else if (toCoreMsg.hasLifecycleEventMsg()) {
forwardToEventService(toCoreMsg.getLifecycleEventMsg(), callback);
} }
} catch (Throwable e) {
log.warn("[{}] Failed to process message: {}", id, msg, e);
callback.onFailure(e);
} }
} });
log.info("TB Core Consumer stopped.");
}); });
if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) {
if (!packSubmitFuture.isDone()) {
packSubmitFuture.cancel(true);
ToCoreMsg lastSubmitMsg = pendingMsgHolder.getToCoreMsg();
log.info("Timeout to process message: {}", lastSubmitMsg);
}
if (log.isDebugEnabled()) {
ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue()));
}
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue()));
}
consumer.commit();
} }
private static class PendingMsgHolder { private static class PendingMsgHolder {
@ -352,7 +347,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
@Override @Override
protected long getNotificationPollDuration() { protected long getNotificationPollDuration() {
return pollDuration; return pollInterval;
} }
@Override @Override
@ -360,6 +355,16 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
return packProcessingTimeout; return packProcessingTimeout;
} }
@Override
protected int getMgmtThreadPoolSize() {
return Math.max(Runtime.getRuntime().availableProcessors(), 4);
}
@Override
protected TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createNotificationsConsumer() {
return queueFactory.createToCoreNotificationsMsgConsumer();
}
@Override @Override
protected void handleNotification(UUID id, TbProtoQueueMsg<ToCoreNotificationMsg> msg, TbCallback callback) { protected void handleNotification(UUID id, TbProtoQueueMsg<ToCoreNotificationMsg> msg, TbCallback callback) {
ToCoreNotificationMsg toCoreNotification = msg.getValue(); ToCoreNotificationMsg toCoreNotification = msg.getValue();
@ -408,92 +413,55 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
} }
} }
private void launchUsageStatsConsumer() { private void processUsageStatsMsg(List<TbProtoQueueMsg<ToUsageStatsServiceMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> consumer) throws Exception {
usageStatsExecutor.submit(() -> { ConcurrentMap<UUID, TbProtoQueueMsg<ToUsageStatsServiceMsg>> pendingMap = msgs.stream().collect(
while (!stopped) { Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
try { CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
List<TbProtoQueueMsg<ToUsageStatsServiceMsg>> msgs = usageStatsConsumer.poll(getNotificationPollDuration()); TbPackProcessingContext<TbProtoQueueMsg<ToUsageStatsServiceMsg>> ctx = new TbPackProcessingContext<>(
if (msgs.isEmpty()) { processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
continue; pendingMap.forEach((id, msg) -> {
} log.trace("[{}] Creating usage stats callback for message: {}", id, msg.getValue());
ConcurrentMap<UUID, TbProtoQueueMsg<ToUsageStatsServiceMsg>> pendingMap = msgs.stream().collect( TbCallback callback = new TbPackCallback<>(id, ctx);
Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); try {
CountDownLatch processingTimeoutLatch = new CountDownLatch(1); handleUsageStats(msg, callback);
TbPackProcessingContext<TbProtoQueueMsg<ToUsageStatsServiceMsg>> ctx = new TbPackProcessingContext<>( } catch (Throwable e) {
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); log.warn("[{}] Failed to process usage stats: {}", id, msg, e);
pendingMap.forEach((id, msg) -> { callback.onFailure(e);
log.trace("[{}] Creating usage stats callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
handleUsageStats(msg, callback);
} catch (Throwable e) {
log.warn("[{}] Failed to process usage stats: {}", id, msg, e);
callback.onFailure(e);
}
});
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process usage stats: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process usage stats: {}", id, msg.getValue()));
}
usageStatsConsumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to obtain usage stats from queue.", e);
try {
Thread.sleep(getNotificationPollDuration());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new usage stats", e2);
}
}
}
} }
log.info("TB Usage Stats Consumer stopped.");
}); });
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process usage stats: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process usage stats: {}", id, msg.getValue()));
}
consumer.commit();
} }
private void launchOtaPackageUpdateNotificationConsumer() { private void processFirmwareMsgs(List<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> consumer) {
long maxProcessingTimeoutPerRecord = firmwarePackInterval / firmwarePackSize; long maxProcessingTimeoutPerRecord = firmwarePackInterval / firmwarePackSize;
firmwareStatesExecutor.submit(() -> { long timeToSleep = maxProcessingTimeoutPerRecord;
while (!stopped) { for (TbProtoQueueMsg<ToOtaPackageStateServiceMsg> msg : msgs) {
try { try {
List<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> msgs = firmwareStatesConsumer.poll(getNotificationPollDuration()); long startTime = System.currentTimeMillis();
if (msgs.isEmpty()) { boolean isSuccessUpdate = handleOtaPackageUpdates(msg);
continue; long endTime = System.currentTimeMillis();
} long spentTime = endTime - startTime;
long timeToSleep = maxProcessingTimeoutPerRecord; timeToSleep = timeToSleep - spentTime;
for (TbProtoQueueMsg<ToOtaPackageStateServiceMsg> msg : msgs) { if (isSuccessUpdate) {
try { if (timeToSleep > 0) {
long startTime = System.currentTimeMillis(); log.debug("Spent time per record is: [{}]!", spentTime);
boolean isSuccessUpdate = handleOtaPackageUpdates(msg); Thread.sleep(timeToSleep);
long endTime = System.currentTimeMillis(); timeToSleep = 0;
long spentTime = endTime - startTime;
timeToSleep = timeToSleep - spentTime;
if (isSuccessUpdate) {
if (timeToSleep > 0) {
log.debug("Spent time per record is: [{}]!", spentTime);
Thread.sleep(timeToSleep);
timeToSleep = 0;
}
timeToSleep += maxProcessingTimeoutPerRecord;
}
} catch (Throwable e) {
log.warn("Failed to process firmware update msg: {}", msg, e);
}
}
firmwareStatesConsumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to obtain usage stats from queue.", e);
try {
Thread.sleep(getNotificationPollDuration());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new firmware updates", e2);
}
} }
timeToSleep += maxProcessingTimeoutPerRecord;
} }
} catch (InterruptedException e) {
return;
} catch (Throwable e) {
log.warn("Failed to process firmware update msg: {}", msg, e);
} }
log.info("TB Ota Package States Consumer stopped."); }
}); consumer.commit();
} }
private void handleUsageStats(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) { private void handleUsageStats(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
@ -781,15 +749,17 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
@Override @Override
protected void stopConsumers() { protected void stopConsumers() {
if (mainConsumer != null) { super.stopConsumers();
mainConsumer.unsubscribe(); mainConsumer.stop();
} mainConsumer.awaitStop();
if (usageStatsConsumer != null) { usageStatsConsumer.stop();
usageStatsConsumer.unsubscribe(); firmwareStatesConsumer.stop();
} }
if (firmwareStatesConsumer != null) {
firmwareStatesConsumer.unsubscribe(); @Data(staticConstructor = "of")
} public static class CoreQueueConfig implements QueueConfig {
private final boolean consumerPerPartition;
private final int pollInterval;
} }
} }

41
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -15,8 +15,8 @@
*/ */
package org.thingsboard.server.service.queue; package org.thingsboard.server.service.queue;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.event.EventListener; import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
@ -39,12 +39,11 @@ import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.QueueDeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.QueueDeleteMsg;
import org.thingsboard.server.gen.transport.TransportProtos.QueueUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos.QueueUpdateMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.queue.util.TbRuleEngineComponent;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbAssetProfileCache;
@ -55,7 +54,6 @@ import org.thingsboard.server.service.queue.ruleengine.TbRuleEngineQueueConsumer
import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService;
import jakarta.annotation.PostConstruct;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
@ -76,7 +74,6 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
private final ConcurrentMap<QueueKey, TbRuleEngineQueueConsumerManager> consumers = new ConcurrentHashMap<>(); private final ConcurrentMap<QueueKey, TbRuleEngineQueueConsumerManager> consumers = new ConcurrentHashMap<>();
public DefaultTbRuleEngineConsumerService(TbRuleEngineConsumerContext ctx, public DefaultTbRuleEngineConsumerService(TbRuleEngineConsumerContext ctx,
TbRuleEngineQueueFactory tbRuleEngineQueueFactory,
ActorSystemContext actorContext, ActorSystemContext actorContext,
TbRuleEngineDeviceRpcService tbDeviceRpcService, TbRuleEngineDeviceRpcService tbDeviceRpcService,
QueueService queueService, QueueService queueService,
@ -87,8 +84,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
PartitionService partitionService, PartitionService partitionService,
ApplicationEventPublisher eventPublisher, ApplicationEventPublisher eventPublisher,
JwtSettingsService jwtSettingsService) { JwtSettingsService jwtSettingsService) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService);
eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), jwtSettingsService);
this.ctx = ctx; this.ctx = ctx;
this.tbDeviceRpcService = tbDeviceRpcService; this.tbDeviceRpcService = tbDeviceRpcService;
this.queueService = queueService; this.queueService = queueService;
@ -96,7 +92,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
@PostConstruct @PostConstruct
public void init() { public void init() {
super.init("tb-rule-engine-notifications-consumer"); super.init("tb-rule-engine");
List<Queue> queues = queueService.findAllQueues(); List<Queue> queues = queueService.findAllQueues();
for (Queue configuration : queues) { for (Queue configuration : queues) {
if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { if (partitionService.isManagedByCurrentService(configuration.getTenantId())) {
@ -136,20 +132,11 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
}); });
} }
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void onApplicationEvent(ApplicationReadyEvent event) {
super.onApplicationEvent(event);
ctx.setReady(true);
}
@Override
protected void launchMainConsumers() {}
@Override @Override
protected void stopConsumers() { protected void stopConsumers() {
super.stopConsumers();
consumers.values().forEach(TbRuleEngineQueueConsumerManager::stop); consumers.values().forEach(TbRuleEngineQueueConsumerManager::stop);
consumers.values().forEach(TbRuleEngineQueueConsumerManager::awaitStop); consumers.values().forEach(TbRuleEngineQueueConsumerManager::awaitStop);
ctx.stop();
} }
@Override @Override
@ -167,6 +154,16 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
return ctx.getPackProcessingTimeout(); return ctx.getPackProcessingTimeout();
} }
@Override
protected int getMgmtThreadPoolSize() {
return ctx.getMgmtThreadPoolSize();
}
@Override
protected TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createNotificationsConsumer() {
return ctx.getQueueFactory().createToRuleEngineNotificationsMsgConsumer();
}
@Override @Override
protected void handleNotification(UUID id, TbProtoQueueMsg<ToRuleEngineNotificationMsg> msg, TbCallback callback) throws Exception { protected void handleNotification(UUID id, TbProtoQueueMsg<ToRuleEngineNotificationMsg> msg, TbCallback callback) throws Exception {
ToRuleEngineNotificationMsg nfMsg = msg.getValue(); ToRuleEngineNotificationMsg nfMsg = msg.getValue();
@ -243,7 +240,13 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
} }
private TbRuleEngineQueueConsumerManager createConsumer(QueueKey queueKey, Queue queue) { private TbRuleEngineQueueConsumerManager createConsumer(QueueKey queueKey, Queue queue) {
var consumer = new TbRuleEngineQueueConsumerManager(ctx, queueKey); var consumer = TbRuleEngineQueueConsumerManager.create()
.ctx(ctx)
.queueKey(queueKey)
.consumerExecutor(consumersExecutor)
.scheduler(scheduler)
.taskExecutor(mgmtExecutor)
.build();
consumers.put(queueKey, consumer); consumers.put(queueKey, consumer);
consumer.init(queue); consumer.init(queue);
return consumer; return consumer;

322
application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java

@ -0,0 +1,322 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue.consumer;
import lombok.Builder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.queue.QueueConfig;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.service.queue.ruleengine.QueueEvent;
import org.thingsboard.server.service.queue.ruleengine.TbQueueConsumerManagerTask;
import org.thingsboard.server.service.queue.ruleengine.TbQueueConsumerTask;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
public class MainQueueConsumerManager<M extends TbQueueMsg, C extends QueueConfig> {
protected final QueueKey queueKey;
@Getter
protected C config;
protected final MsgPackProcessor<M, C> msgPackProcessor;
protected final Function<C, TbQueueConsumer<M>> consumerCreator;
protected final ExecutorService consumerExecutor;
protected final ScheduledExecutorService scheduler;
protected final ExecutorService taskExecutor;
private final java.util.Queue<TbQueueConsumerManagerTask> tasks = new ConcurrentLinkedQueue<>();
private final ReentrantLock lock = new ReentrantLock();
@Getter
private volatile Set<TopicPartitionInfo> partitions;
protected volatile ConsumerWrapper<M> consumerWrapper;
protected volatile boolean stopped;
@Builder
public MainQueueConsumerManager(QueueKey queueKey, C config,
MsgPackProcessor<M, C> msgPackProcessor,
Function<C, TbQueueConsumer<M>> consumerCreator,
ExecutorService consumerExecutor,
ScheduledExecutorService scheduler,
ExecutorService taskExecutor) {
this.queueKey = queueKey;
this.config = config;
this.msgPackProcessor = msgPackProcessor;
this.consumerCreator = consumerCreator;
this.consumerExecutor = consumerExecutor;
this.scheduler = scheduler;
this.taskExecutor = taskExecutor;
if (config != null) {
init(config);
}
}
public void init(C config) {
this.config = config;
if (config.isConsumerPerPartition()) {
this.consumerWrapper = new ConsumerPerPartitionWrapper();
} else {
this.consumerWrapper = new SingleConsumerWrapper();
}
log.debug("[{}] Initialized consumer for queue: {}", queueKey, config);
}
public void update(C config) {
addTask(TbQueueConsumerManagerTask.configUpdate(config));
}
public void update(Set<TopicPartitionInfo> partitions) {
addTask(TbQueueConsumerManagerTask.partitionChange(partitions));
}
protected void addTask(TbQueueConsumerManagerTask todo) {
if (stopped) {
return;
}
tasks.add(todo);
log.trace("[{}] Added task: {}", queueKey, todo);
tryProcessTasks();
}
private void tryProcessTasks() {
taskExecutor.submit(() -> {
if (lock.tryLock()) {
try {
C newConfig = null;
Set<TopicPartitionInfo> newPartitions = null;
while (!stopped) {
TbQueueConsumerManagerTask task = tasks.poll();
if (task == null) {
break;
}
log.trace("[{}] Processing task: {}", queueKey, task);
if (task.getEvent() == QueueEvent.PARTITION_CHANGE) {
newPartitions = task.getPartitions();
} else if (task.getEvent() == QueueEvent.CONFIG_UPDATE) {
newConfig = (C) task.getConfig();
} else {
processTask(task);
}
}
if (stopped) {
return;
}
if (newConfig != null) {
doUpdate(newConfig);
}
if (newPartitions != null) {
doUpdate(newPartitions);
}
} catch (Exception e) {
log.error("[{}] Failed to process tasks", queueKey, e);
} finally {
lock.unlock();
}
} else {
log.trace("[{}] Failed to acquire lock", queueKey);
scheduler.schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS);
}
});
}
protected void processTask(TbQueueConsumerManagerTask task) {
}
private void doUpdate(C newConfig) {
log.info("[{}] Processing queue update: {}", queueKey, newConfig);
var oldConfig = this.config;
this.config = newConfig;
if (log.isTraceEnabled()) {
log.trace("[{}] Old queue configuration: {}", queueKey, oldConfig);
log.trace("[{}] New queue configuration: {}", queueKey, newConfig);
}
if (oldConfig == null) {
init(config);
} else if (newConfig.isConsumerPerPartition() != oldConfig.isConsumerPerPartition()) {
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop);
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion);
init(config);
if (partitions != null) {
doUpdate(partitions); // even if partitions number was changed, there can be no partition change event
}
} else {
log.trace("[{}] Silently applied new config, because consumer-per-partition not changed", queueKey);
// do nothing, because partitions change (if they changed) will be handled on PartitionChangeEvent,
// and changes to other config values will be picked up by consumer on the fly,
// and queue topic and name are immutable
}
}
private void doUpdate(Set<TopicPartitionInfo> partitions) {
this.partitions = partitions;
consumerWrapper.updatePartitions(partitions);
}
private void launchConsumer(TbQueueConsumerTask<M> consumerTask) {
log.info("[{}] Launching consumer", consumerTask.getKey());
Future<?> consumerLoop = consumerExecutor.submit(() -> {
ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString());
try {
consumerLoop(consumerTask.getConsumer());
} catch (Throwable e) {
log.error("Failure in consumer loop", e);
}
log.info("[{}] Consumer stopped", consumerTask.getKey());
});
consumerTask.setTask(consumerLoop);
}
private void consumerLoop(TbQueueConsumer<M> consumer) {
while (!stopped && !consumer.isStopped()) {
try {
List<M> msgs = consumer.poll(config.getPollInterval());
if (msgs.isEmpty()) {
continue;
}
processMsgs(msgs, consumer, config);
} catch (Exception e) {
if (!consumer.isStopped()) {
log.warn("Failed to process messages from queue", e);
try {
Thread.sleep(config.getPollInterval());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new requests", e2);
}
}
}
}
if (consumer.isStopped()) {
consumer.unsubscribe();
}
}
protected void processMsgs(List<M> msgs, TbQueueConsumer<M> consumer, C config) throws Exception {
msgPackProcessor.process(msgs, consumer, config);
}
public void stop() {
log.debug("[{}] Stopping consumers", queueKey);
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop);
stopped = true;
}
public void awaitStop() {
log.debug("[{}] Waiting for consumers to stop", queueKey);
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion);
log.debug("[{}] Unsubscribed and stopped consumers", queueKey);
}
private static String partitionsToString(Collection<TopicPartitionInfo> partitions) {
return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.joining(", ", "[", "]"));
}
public interface MsgPackProcessor<M extends TbQueueMsg, C extends QueueConfig> {
void process(List<M> msgs, TbQueueConsumer<M> consumer, C config) throws Exception;
}
public interface ConsumerWrapper<M extends TbQueueMsg> {
void updatePartitions(Set<TopicPartitionInfo> partitions);
Collection<TbQueueConsumerTask<M>> getConsumers();
}
class ConsumerPerPartitionWrapper implements ConsumerWrapper<M> {
private final Map<TopicPartitionInfo, TbQueueConsumerTask<M>> consumers = new HashMap<>();
@Override
public void updatePartitions(Set<TopicPartitionInfo> partitions) {
Set<TopicPartitionInfo> addedPartitions = new HashSet<>(partitions);
addedPartitions.removeAll(consumers.keySet());
Set<TopicPartitionInfo> removedPartitions = new HashSet<>(consumers.keySet());
removedPartitions.removeAll(partitions);
log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, partitionsToString(addedPartitions), partitionsToString(removedPartitions));
removedPartitions.forEach((tpi) -> consumers.get(tpi).initiateStop());
removedPartitions.forEach((tpi) -> consumers.remove(tpi).awaitCompletion());
addedPartitions.forEach((tpi) -> {
String key = queueKey + "-" + tpi.getPartition().orElse(-1);
TbQueueConsumerTask<M> consumer = new TbQueueConsumerTask<>(key, consumerCreator.apply(config));
consumers.put(tpi, consumer);
consumer.subscribe(Set.of(tpi));
launchConsumer(consumer);
});
}
@Override
public Collection<TbQueueConsumerTask<M>> getConsumers() {
return consumers.values();
}
}
class SingleConsumerWrapper implements ConsumerWrapper<M> {
private TbQueueConsumerTask<M> consumer;
@Override
public void updatePartitions(Set<TopicPartitionInfo> partitions) {
log.info("[{}] New partitions: {}", queueKey, partitionsToString(partitions));
if (partitions.isEmpty()) {
if (consumer != null && consumer.isRunning()) {
consumer.initiateStop();
consumer.awaitCompletion();
}
consumer = null;
return;
}
if (consumer == null) {
consumer = new TbQueueConsumerTask<>(queueKey, consumerCreator.apply(config));
}
consumer.subscribe(partitions);
if (!consumer.isRunning()) {
launchConsumer(consumer);
}
}
@Override
public Collection<TbQueueConsumerTask<M>> getConsumers() {
if (consumer == null) {
return Collections.emptyList();
}
return List.of(consumer);
}
}
}

131
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@ -15,10 +15,11 @@
*/ */
package org.thingsboard.server.service.queue.processing; package org.thingsboard.server.service.queue.processing;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.ApplicationEventPublisher;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
@ -36,6 +37,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
@ -47,7 +49,6 @@ import org.thingsboard.server.service.queue.TbPackCallback;
import org.thingsboard.server.service.queue.TbPackProcessingContext; import org.thingsboard.server.service.queue.TbPackProcessingContext;
import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService;
import jakarta.annotation.PreDestroy;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
@ -55,6 +56,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -62,9 +64,6 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor @RequiredArgsConstructor
public abstract class AbstractConsumerService<N extends com.google.protobuf.GeneratedMessageV3> extends TbApplicationEventListener<PartitionChangeEvent> { public abstract class AbstractConsumerService<N extends com.google.protobuf.GeneratedMessageV3> extends TbApplicationEventListener<PartitionChangeEvent> {
protected volatile ExecutorService notificationsConsumerExecutor;
protected volatile boolean stopped = false;
protected volatile boolean isReady = false;
protected final ActorSystemContext actorContext; protected final ActorSystemContext actorContext;
protected final TbTenantProfileCache tenantProfileCache; protected final TbTenantProfileCache tenantProfileCache;
protected final TbDeviceProfileCache deviceProfileCache; protected final TbDeviceProfileCache deviceProfileCache;
@ -72,21 +71,37 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
protected final TbApiUsageStateService apiUsageStateService; protected final TbApiUsageStateService apiUsageStateService;
protected final PartitionService partitionService; protected final PartitionService partitionService;
protected final ApplicationEventPublisher eventPublisher; protected final ApplicationEventPublisher eventPublisher;
protected final TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer;
protected final JwtSettingsService jwtSettingsService; protected final JwtSettingsService jwtSettingsService;
public void init(String nfConsumerThreadName) { protected QueueConsumerManager<TbProtoQueueMsg<N>> nfConsumer;
this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(nfConsumerThreadName));
protected ExecutorService consumersExecutor;
protected ExecutorService mgmtExecutor;
protected ScheduledExecutorService scheduler;
public void init(String prefix) {
this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName(prefix + "-consumer"));
this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(getMgmtThreadPoolSize(), prefix + "-mgmt");
this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName(prefix + "-consumer-scheduler"));
this.nfConsumer = QueueConsumerManager.<TbProtoQueueMsg<N>>builder()
.name(getServiceType().getLabel() + " Notifications")
.msgPackProcessor(this::processNotifications)
.pollInterval(getNotificationPollDuration())
.consumerCreator(this::createNotificationsConsumer)
.consumerExecutor(consumersExecutor)
.threadPrefix("notifications")
.build();
} }
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void onApplicationEvent(ApplicationReadyEvent event) { public void afterStartUp() {
log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); startConsumers();
this.nfConsumer.subscribe(); }
this.isReady = true;
launchNotificationsConsumer(); protected void startConsumers() {
launchMainConsumers(); nfConsumer.subscribe();
nfConsumer.launch();
} }
@Override @Override
@ -96,58 +111,42 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
protected abstract ServiceType getServiceType(); protected abstract ServiceType getServiceType();
protected abstract void launchMainConsumers(); protected void stopConsumers() {
nfConsumer.stop();
protected abstract void stopConsumers(); }
protected abstract long getNotificationPollDuration(); protected abstract long getNotificationPollDuration();
protected abstract long getNotificationPackProcessingTimeout(); protected abstract long getNotificationPackProcessingTimeout();
protected void launchNotificationsConsumer() { protected abstract int getMgmtThreadPoolSize();
notificationsConsumerExecutor.submit(() -> {
while (!stopped) { protected abstract TbQueueConsumer<TbProtoQueueMsg<N>> createNotificationsConsumer();
try {
List<TbProtoQueueMsg<N>> msgs = nfConsumer.poll(getNotificationPollDuration()); protected void processNotifications(List<TbProtoQueueMsg<N>> msgs, TbQueueConsumer<TbProtoQueueMsg<N>> consumer) throws Exception {
if (msgs.isEmpty()) { List<IdMsgPair<N>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).collect(Collectors.toList());
continue; ConcurrentMap<UUID, TbProtoQueueMsg<N>> pendingMap = orderedMsgList.stream().collect(
} Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
List<IdMsgPair<N>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).collect(Collectors.toList()); CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
ConcurrentMap<UUID, TbProtoQueueMsg<N>> pendingMap = orderedMsgList.stream().collect( TbPackProcessingContext<TbProtoQueueMsg<N>> ctx = new TbPackProcessingContext<>(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
CountDownLatch processingTimeoutLatch = new CountDownLatch(1); orderedMsgList.forEach(element -> {
TbPackProcessingContext<TbProtoQueueMsg<N>> ctx = new TbPackProcessingContext<>( UUID id = element.getUuid();
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); TbProtoQueueMsg<N> msg = element.getMsg();
orderedMsgList.forEach(element -> { log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue());
UUID id = element.getUuid(); TbCallback callback = new TbPackCallback<>(id, ctx);
TbProtoQueueMsg<N> msg = element.getMsg(); try {
log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); handleNotification(id, msg, callback);
TbCallback callback = new TbPackCallback<>(id, ctx); } catch (Throwable e) {
try { log.warn("[{}] Failed to process notification: {}", id, msg, e);
handleNotification(id, msg, callback); callback.onFailure(e);
} catch (Throwable e) {
log.warn("[{}] Failed to process notification: {}", id, msg, e);
callback.onFailure(e);
}
});
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue()));
}
nfConsumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to obtain notifications from queue.", e);
try {
Thread.sleep(getNotificationPollDuration());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new notifications", e2);
}
}
}
} }
log.info("TB Notifications Consumer stopped.");
}); });
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue()));
}
consumer.commit();
} }
protected final void handleComponentLifecycleMsg(UUID id, ComponentLifecycleMsg componentLifecycleMsg) { protected final void handleComponentLifecycleMsg(UUID id, ComponentLifecycleMsg componentLifecycleMsg) {
@ -201,13 +200,15 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
@PreDestroy @PreDestroy
public void destroy() { public void destroy() {
stopped = true;
stopConsumers(); stopConsumers();
if (nfConsumer != null) { if (consumersExecutor != null) {
nfConsumer.unsubscribe(); consumersExecutor.shutdownNow();
}
if (mgmtExecutor != null) {
mgmtExecutor.shutdownNow();
} }
if (notificationsConsumerExecutor != null) { if (scheduler != null) {
notificationsConsumerExecutor.shutdownNow(); scheduler.shutdownNow();
} }
} }
} }

8
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java

@ -18,7 +18,7 @@ package org.thingsboard.server.service.queue.ruleengine;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Getter; import lombok.Getter;
import lombok.ToString; import lombok.ToString;
import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.QueueConfig;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import java.util.Set; import java.util.Set;
@ -29,7 +29,7 @@ import java.util.Set;
public class TbQueueConsumerManagerTask { public class TbQueueConsumerManagerTask {
private final QueueEvent event; private final QueueEvent event;
private Queue queue; private QueueConfig config;
private Set<TopicPartitionInfo> partitions; private Set<TopicPartitionInfo> partitions;
private boolean drainQueue; private boolean drainQueue;
@ -37,8 +37,8 @@ public class TbQueueConsumerManagerTask {
return new TbQueueConsumerManagerTask(QueueEvent.DELETE, null, null, drainQueue); return new TbQueueConsumerManagerTask(QueueEvent.DELETE, null, null, drainQueue);
} }
public static TbQueueConsumerManagerTask configUpdate(Queue queue) { public static TbQueueConsumerManagerTask configUpdate(QueueConfig config) {
return new TbQueueConsumerManagerTask(QueueEvent.CONFIG_UPDATE, queue, null, false); return new TbQueueConsumerManagerTask(QueueEvent.CONFIG_UPDATE, config, null, false);
} }
public static TbQueueConsumerManagerTask partitionChange(Set<TopicPartitionInfo> partitions) { public static TbQueueConsumerManagerTask partitionChange(Set<TopicPartitionInfo> partitions) {

7
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java

@ -20,9 +20,8 @@ import lombok.RequiredArgsConstructor;
import lombok.Setter; import lombok.Setter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.TbQueueMsg;
import java.util.Set; import java.util.Set;
import java.util.concurrent.Future; import java.util.concurrent.Future;
@ -30,12 +29,12 @@ import java.util.concurrent.TimeUnit;
@RequiredArgsConstructor @RequiredArgsConstructor
@Slf4j @Slf4j
public class TbQueueConsumerTask { public class TbQueueConsumerTask<M extends TbQueueMsg> {
@Getter @Getter
private final Object key; private final Object key;
@Getter @Getter
private final TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> consumer; private final TbQueueConsumer<M> consumer;
@Setter @Setter
private Future<?> task; private Future<?> task;

25
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java

@ -19,8 +19,6 @@ import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -33,11 +31,6 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStr
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory;
import org.thingsboard.server.service.stats.RuleEngineStatisticsService; import org.thingsboard.server.service.stats.RuleEngineStatisticsService;
import javax.annotation.PostConstruct;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@Component @Component
@TbRuleEngineComponent @TbRuleEngineComponent
@Slf4j @Slf4j
@ -68,22 +61,4 @@ public class TbRuleEngineConsumerContext {
private final TbQueueProducerProvider producerProvider; private final TbQueueProducerProvider producerProvider;
private final TbQueueAdmin queueAdmin; private final TbQueueAdmin queueAdmin;
private ExecutorService consumersExecutor;
private ExecutorService mgmtExecutor;
private ScheduledExecutorService scheduler;
private volatile boolean isReady = false;
@PostConstruct
void init() {
this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer"));
this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(mgmtThreadPoolSize, "tb-rule-engine-mgmt");
this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler"));
}
public void stop() {
scheduler.shutdownNow();
consumersExecutor.shutdownNow();
mgmtExecutor.shutdownNow();
}
} }

290
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java

@ -16,9 +16,8 @@
package org.thingsboard.server.service.queue.ruleengine; package org.thingsboard.server.service.queue.ruleengine;
import com.google.protobuf.ProtocolStringList; import com.google.protobuf.ProtocolStringList;
import lombok.Getter; import lombok.Builder;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
@ -38,173 +37,54 @@ import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackCallback;
import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext;
import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats;
import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Slf4j @Slf4j
public class TbRuleEngineQueueConsumerManager { public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<TbProtoQueueMsg<ToRuleEngineMsg>, Queue> {
public static final String SUCCESSFUL_STATUS = "successful"; public static final String SUCCESSFUL_STATUS = "successful";
public static final String FAILED_STATUS = "failed"; public static final String FAILED_STATUS = "failed";
private final TbRuleEngineConsumerContext ctx; private final TbRuleEngineConsumerContext ctx;
private final QueueKey queueKey;
private final TbRuleEngineConsumerStats stats; private final TbRuleEngineConsumerStats stats;
private final ReentrantLock lock = new ReentrantLock(); //NonfairSync
@Getter @Builder(builderMethodName = "create") // not to conflict with super.builder()
private volatile Queue queue; public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx,
@Getter QueueKey queueKey,
private volatile Set<TopicPartitionInfo> partitions; ExecutorService consumerExecutor,
private volatile ConsumerWrapper consumerWrapper; ScheduledExecutorService scheduler,
ExecutorService taskExecutor) {
private volatile boolean stopped; super(queueKey, null, null, ctx.getQueueFactory()::createToRuleEngineMsgConsumer, consumerExecutor, scheduler, taskExecutor);
private final java.util.Queue<TbQueueConsumerManagerTask> tasks = new ConcurrentLinkedQueue<>();
public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx, QueueKey queueKey) {
this.ctx = ctx; this.ctx = ctx;
this.queueKey = queueKey;
this.stats = new TbRuleEngineConsumerStats(queueKey, ctx.getStatsFactory()); this.stats = new TbRuleEngineConsumerStats(queueKey, ctx.getStatsFactory());
} }
public void init(Queue queue) {
this.queue = queue;
if (queue.isConsumerPerPartition()) {
this.consumerWrapper = new ConsumerPerPartitionWrapper();
} else {
this.consumerWrapper = new SingleConsumerWrapper();
}
log.debug("[{}] Initialized consumer for queue: {}", queueKey, queue);
}
public void update(Queue queue) {
addTask(TbQueueConsumerManagerTask.configUpdate(queue));
}
public void update(Set<TopicPartitionInfo> partitions) {
addTask(TbQueueConsumerManagerTask.partitionChange(partitions));
}
public void delete(boolean drainQueue) { public void delete(boolean drainQueue) {
addTask(TbQueueConsumerManagerTask.delete(drainQueue)); addTask(TbQueueConsumerManagerTask.delete(drainQueue));
} }
private void addTask(TbQueueConsumerManagerTask todo) { @Override
if (stopped) { protected void processTask(TbQueueConsumerManagerTask task) {
return; if (task.getEvent() == QueueEvent.DELETE) {
} doDelete(task.isDrainQueue());
tasks.add(todo);
log.trace("[{}] Added task: {}", queueKey, todo);
tryProcessTasks();
}
private void tryProcessTasks() {
if (!ctx.isReady()) {
log.debug("[{}] TbRuleEngineConsumerContext is not ready yet, will process tasks later", queueKey);
ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS);
return;
}
ctx.getMgmtExecutor().submit(() -> {
if (lock.tryLock()) {
try {
Queue newConfiguration = null;
Set<TopicPartitionInfo> newPartitions = null;
while (!stopped) {
TbQueueConsumerManagerTask task = tasks.poll();
if (task == null) {
break;
}
log.trace("[{}] Processing task: {}", queueKey, task);
if (task.getEvent() == QueueEvent.PARTITION_CHANGE) {
newPartitions = task.getPartitions();
} else if (task.getEvent() == QueueEvent.CONFIG_UPDATE) {
newConfiguration = task.getQueue();
} else if (task.getEvent() == QueueEvent.DELETE) {
doDelete(task.isDrainQueue());
return;
}
}
if (stopped) {
return;
}
if (newConfiguration != null) {
doUpdate(newConfiguration);
}
if (newPartitions != null) {
doUpdate(newPartitions);
}
} catch (Exception e) {
log.error("[{}] Failed to process tasks", queueKey, e);
} finally {
lock.unlock();
}
} else {
log.trace("[{}] Failed to acquire lock", queueKey);
ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS);
}
});
}
private void doUpdate(Queue newQueue) {
log.info("[{}] Processing queue update: {}", queueKey, newQueue);
var oldQueue = this.queue;
this.queue = newQueue;
if (log.isTraceEnabled()) {
log.trace("[{}] Old queue configuration: {}", queueKey, oldQueue);
log.trace("[{}] New queue configuration: {}", queueKey, newQueue);
}
if (oldQueue == null) {
init(queue);
} else if (newQueue.isConsumerPerPartition() != oldQueue.isConsumerPerPartition()) {
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop);
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion);
init(queue);
if (partitions != null) {
doUpdate(partitions); // even if partitions number was changed, there can be no partition change event
}
} else {
// do nothing, because partitions change (if they changed) will be handled on PartitionChangeEvent,
// and changes to pollInterval/packProcessingTimeout/submitStrategy/processingStrategy will be picked up by consumer on the fly,
// and queue topic and name are immutable
} }
} }
private void doUpdate(Set<TopicPartitionInfo> partitions) {
this.partitions = partitions;
consumerWrapper.updatePartitions(partitions);
}
public void stop() {
log.debug("[{}] Stopping consumers", queueKey);
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop);
stopped = true;
}
public void awaitStop() {
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion);
log.debug("[{}] Unsubscribed and stopped consumers", queueKey);
}
private void doDelete(boolean drainQueue) { private void doDelete(boolean drainQueue) {
stopped = true; stopped = true;
log.info("[{}] Handling queue deletion", queueKey); log.info("[{}] Handling queue deletion", queueKey);
@ -212,7 +92,7 @@ public class TbRuleEngineQueueConsumerManager {
List<TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>>> queueConsumers = consumerWrapper.getConsumers().stream() List<TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>>> queueConsumers = consumerWrapper.getConsumers().stream()
.map(TbQueueConsumerTask::getConsumer).collect(Collectors.toList()); .map(TbQueueConsumerTask::getConsumer).collect(Collectors.toList());
ctx.getConsumersExecutor().submit(() -> { consumerExecutor.submit(() -> {
if (drainQueue) { if (drainQueue) {
drainQueue(queueConsumers); drainQueue(queueConsumers);
} }
@ -235,47 +115,10 @@ public class TbRuleEngineQueueConsumerManager {
}); });
} }
private void launchConsumer(TbQueueConsumerTask consumerTask) { @Override
log.info("[{}] Launching consumer", consumerTask.getKey()); protected void processMsgs(List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs,
Future<?> consumerLoop = ctx.getConsumersExecutor().submit(() -> { TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer,
ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); Queue queue) throws Exception {
try {
consumerLoop(consumerTask.getConsumer());
} catch (Throwable e) {
log.error("Failure in consumer loop", e);
}
});
consumerTask.setTask(consumerLoop);
}
private void consumerLoop(TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer) {
while (!stopped && !consumer.isStopped()) {
try {
List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs = consumer.poll(queue.getPollInterval());
if (msgs.isEmpty()) {
continue;
}
processMsgs(msgs, consumer, queue);
} catch (Exception e) {
if (!consumer.isStopped()) {
log.warn("Failed to process messages from queue", e);
try {
Thread.sleep(ctx.getPollDuration());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new requests", e2);
}
}
}
}
if (consumer.isStopped()) {
consumer.unsubscribe();
}
log.info("Rule Engine consumer stopped");
}
private void processMsgs(List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs,
TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer,
Queue queue) throws InterruptedException {
TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(queue); TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(queue);
TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue); TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue);
submitStrategy.init(msgs); submitStrategy.init(msgs);
@ -320,7 +163,7 @@ public class TbRuleEngineQueueConsumerManager {
} }
private void submitMessage(TbMsgPackProcessingContext packCtx, UUID id, TbProtoQueueMsg<ToRuleEngineMsg> msg) { private void submitMessage(TbMsgPackProcessingContext packCtx, UUID id, TbProtoQueueMsg<ToRuleEngineMsg> msg) {
log.trace("[{}] Creating callback for topic {} message: {}", id, queue.getName(), msg.getValue()); log.trace("[{}] Creating callback for topic {} message: {}", id, config.getName(), msg.getValue());
ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); ToRuleEngineMsg toRuleEngineMsg = msg.getValue();
TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB()));
TbMsgCallback callback = ctx.isPrometheusStatsEnabled() ? TbMsgCallback callback = ctx.isPrometheusStatsEnabled() ?
@ -328,7 +171,7 @@ public class TbRuleEngineQueueConsumerManager {
new TbMsgPackCallback(id, tenantId, packCtx); new TbMsgPackCallback(id, tenantId, packCtx);
try { try {
if (!toRuleEngineMsg.getTbMsg().isEmpty()) { if (!toRuleEngineMsg.getTbMsg().isEmpty()) {
forwardToRuleEngineActor(queue.getName(), tenantId, toRuleEngineMsg, callback); forwardToRuleEngineActor(config.getName(), tenantId, toRuleEngineMsg, callback);
} else { } else {
callback.onSuccess(); callback.onSuccess();
} }
@ -356,7 +199,7 @@ public class TbRuleEngineQueueConsumerManager {
log.info("[{}] {} to process [{}] messages", queueKey, prefix, map.size()); log.info("[{}] {} to process [{}] messages", queueKey, prefix, map.size());
for (Map.Entry<UUID, TbProtoQueueMsg<ToRuleEngineMsg>> pending : map.entrySet()) { for (Map.Entry<UUID, TbProtoQueueMsg<ToRuleEngineMsg>> pending : map.entrySet()) {
ToRuleEngineMsg tmp = pending.getValue().getValue(); ToRuleEngineMsg tmp = pending.getValue().getValue();
TbMsg tmpMsg = TbMsg.fromBytes(queue.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); TbMsg tmpMsg = TbMsg.fromBytes(config.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY);
RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey()); RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey());
if (printAll) { if (printAll) {
log.trace("[{}][{}] {} to process message: {}, Last Rule Node: {}", queueKey, TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); log.trace("[{}][{}] {} to process message: {}, Last Rule Node: {}", queueKey, TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo);
@ -379,7 +222,7 @@ public class TbRuleEngineQueueConsumerManager {
int n = 0; int n = 0;
while (System.currentTimeMillis() <= finishTs) { while (System.currentTimeMillis() <= finishTs) {
for (TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer : consumers) { for (TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer : consumers) {
List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs = consumer.poll(queue.getPollInterval()); List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs = consumer.poll(config.getPollInterval());
if (msgs.isEmpty()) { if (msgs.isEmpty()) {
continue; continue;
} }
@ -388,7 +231,7 @@ public class TbRuleEngineQueueConsumerManager {
MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray());
EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB()));
TopicPartitionInfo tpi = ctx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); TopicPartitionInfo tpi = ctx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, config.getName(), TenantId.SYS_TENANT_ID, originator);
ctx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, msg, null); ctx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, msg, null);
n++; n++;
} catch (Throwable e) { } catch (Throwable e) {
@ -399,90 +242,11 @@ public class TbRuleEngineQueueConsumerManager {
} }
} }
if (n > 0) { if (n > 0) {
log.info("Moved {} messages from {} to system {}", n, queueKey, queue.getName()); log.info("Moved {} messages from {} to system {}", n, queueKey, config.getName());
} }
} catch (Exception e) { } catch (Exception e) {
log.error("[{}] Failed to drain queue", queueKey, e); log.error("[{}] Failed to drain queue", queueKey, e);
} }
} }
private static String partitionsToString(Collection<TopicPartitionInfo> partitions) {
return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.joining(", ", "[", "]"));
}
interface ConsumerWrapper {
void updatePartitions(Set<TopicPartitionInfo> partitions);
Collection<TbQueueConsumerTask> getConsumers();
}
class ConsumerPerPartitionWrapper implements ConsumerWrapper {
private final Map<TopicPartitionInfo, TbQueueConsumerTask> consumers = new HashMap<>();
@Override
public void updatePartitions(Set<TopicPartitionInfo> partitions) {
Set<TopicPartitionInfo> addedPartitions = new HashSet<>(partitions);
addedPartitions.removeAll(consumers.keySet());
Set<TopicPartitionInfo> removedPartitions = new HashSet<>(consumers.keySet());
removedPartitions.removeAll(partitions);
log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, partitionsToString(addedPartitions), partitionsToString(removedPartitions));
removedPartitions.forEach((tpi) -> {
consumers.get(tpi).initiateStop();
});
removedPartitions.forEach((tpi) -> {
consumers.remove(tpi).awaitCompletion();
});
addedPartitions.forEach((tpi) -> {
String key = queueKey + "-" + tpi.getPartition().orElse(-999999);
TbQueueConsumerTask consumer = new TbQueueConsumerTask(key, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue));
consumers.put(tpi, consumer);
consumer.subscribe(Set.of(tpi));
launchConsumer(consumer);
});
}
@Override
public Collection<TbQueueConsumerTask> getConsumers() {
return consumers.values();
}
}
class SingleConsumerWrapper implements ConsumerWrapper {
private TbQueueConsumerTask consumer;
@Override
public void updatePartitions(Set<TopicPartitionInfo> partitions) {
log.info("[{}] New partitions: {}", queueKey, partitionsToString(partitions));
if (partitions.isEmpty()) {
if (consumer != null && consumer.isRunning()) {
consumer.initiateStop();
consumer.awaitCompletion();
}
consumer = null;
return;
}
if (consumer == null) {
consumer = new TbQueueConsumerTask(queueKey, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue));
}
consumer.subscribe(partitions);
if (!consumer.isRunning()) {
launchConsumer(consumer);
}
}
@Override
public Collection<TbQueueConsumerTask> getConsumers() {
if (consumer == null) {
return Collections.emptyList();
}
return List.of(consumer);
}
}
} }

80
application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsService.java

@ -17,25 +17,24 @@ package org.thingsboard.server.service.security.auth.jwt.settings;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.notification.targets.platform.SystemAdministratorsFilter;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.dao.notification.DefaultNotifications;
import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import java.nio.charset.StandardCharsets;
import java.util.Base64; import java.util.Base64;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import static org.thingsboard.server.service.security.model.token.JwtTokenFactory.KEY_LENGTH;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@Slf4j @Slf4j
@ -43,49 +42,11 @@ public class DefaultJwtSettingsService implements JwtSettingsService {
private final AdminSettingsService adminSettingsService; private final AdminSettingsService adminSettingsService;
private final Optional<TbClusterService> tbClusterService; private final Optional<TbClusterService> tbClusterService;
private final Optional<NotificationCenter> notificationCenter;
private final JwtSettingsValidator jwtSettingsValidator; private final JwtSettingsValidator jwtSettingsValidator;
private final Optional<JwtTokenFactory> jwtTokenFactory;
@Value("${security.jwt.tokenExpirationTime:9000}")
private Integer tokenExpirationTime;
@Value("${security.jwt.refreshTokenExpTime:604800}")
private Integer refreshTokenExpTime;
@Value("${security.jwt.tokenIssuer:thingsboard.io}")
private String tokenIssuer;
@Value("${security.jwt.tokenSigningKey:thingsboardDefaultSigningKey}")
private String tokenSigningKey;
private volatile JwtSettings jwtSettings = null; //lazy init private volatile JwtSettings jwtSettings = null; //lazy init
/**
* Create JWT admin settings is intended to be called from Install scripts only
*/
@Override
public void createRandomJwtSettings() {
if (getJwtSettingsFromDb() == null) {
log.info("Creating JWT admin settings...");
this.jwtSettings = getJwtSettingsFromYml();
if (isSigningKeyDefault(jwtSettings)) {
this.jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(
RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8)));
}
saveJwtSettings(jwtSettings);
} else {
log.info("Skip creating JWT admin settings because they already exist.");
}
}
/**
* Create JWT admin settings is intended to be called from Upgrade scripts only
*/
@Override
public void saveLegacyYmlSettings() {
log.info("Saving legacy JWT admin settings from YML...");
if (getJwtSettingsFromDb() == null) {
saveJwtSettings(getJwtSettingsFromYml());
}
}
@Override @Override
public JwtSettings saveJwtSettings(JwtSettings jwtSettings) { public JwtSettings saveJwtSettings(JwtSettings jwtSettings) {
jwtSettingsValidator.validate(jwtSettings); jwtSettingsValidator.validate(jwtSettings);
@ -105,7 +66,9 @@ public class DefaultJwtSettingsService implements JwtSettingsService {
@Override @Override
public JwtSettings reloadJwtSettings() { public JwtSettings reloadJwtSettings() {
log.trace("Executing reloadJwtSettings"); log.trace("Executing reloadJwtSettings");
return getJwtSettings(true); var settings = getJwtSettings(true);
jwtTokenFactory.ifPresent(JwtTokenFactory::reload);
return settings;
} }
@Override @Override
@ -118,30 +81,13 @@ public class DefaultJwtSettingsService implements JwtSettingsService {
if (this.jwtSettings == null || forceReload) { if (this.jwtSettings == null || forceReload) {
synchronized (this) { synchronized (this) {
if (this.jwtSettings == null || forceReload) { if (this.jwtSettings == null || forceReload) {
JwtSettings result = getJwtSettingsFromDb(); jwtSettings = getJwtSettingsFromDb();
if (result == null) {
result = getJwtSettingsFromYml();
log.warn("Loading the JWT settings from YML since there are no settings in DB. Looks like the upgrade script was not applied.");
}
if (isSigningKeyDefault(result)) {
log.warn("WARNING: The platform is configured to use default JWT Signing Key. " +
"This is a security issue that needs to be resolved. Please change the JWT Signing Key using the Web UI. " +
"Navigate to \"System settings -> Security settings\" while logged in as a System Administrator.");
notificationCenter.ifPresent(notificationCenter -> {
notificationCenter.sendGeneralWebNotification(TenantId.SYS_TENANT_ID, new SystemAdministratorsFilter(), DefaultNotifications.jwtSigningKeyIssue.toTemplate());
});
}
this.jwtSettings = result;
} }
} }
} }
return this.jwtSettings; return this.jwtSettings;
} }
private JwtSettings getJwtSettingsFromYml() {
return new JwtSettings(this.tokenExpirationTime, this.refreshTokenExpTime, this.tokenIssuer, this.tokenSigningKey);
}
private JwtSettings getJwtSettingsFromDb() { private JwtSettings getJwtSettingsFromDb() {
AdminSettings adminJwtSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); AdminSettings adminJwtSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY);
return adminJwtSettings != null ? mapAdminToJwtSettings(adminJwtSettings) : null; return adminJwtSettings != null ? mapAdminToJwtSettings(adminJwtSettings) : null;
@ -161,8 +107,12 @@ public class DefaultJwtSettingsService implements JwtSettingsService {
return adminJwtSettings; return adminJwtSettings;
} }
private boolean isSigningKeyDefault(JwtSettings settings) { public static boolean isSigningKeyDefault(JwtSettings settings) {
return TOKEN_SIGNING_KEY_DEFAULT.equals(settings.getTokenSigningKey()); return TOKEN_SIGNING_KEY_DEFAULT.equals(settings.getTokenSigningKey());
} }
public static boolean validateTokenSigningKeyLength(JwtSettings settings) {
return Base64.getDecoder().decode(settings.getTokenSigningKey()).length * Byte.SIZE >= KEY_LENGTH;
}
} }

7
application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java

@ -27,6 +27,9 @@ import java.util.Base64;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static org.thingsboard.server.service.security.auth.jwt.settings.DefaultJwtSettingsService.isSigningKeyDefault;
import static org.thingsboard.server.service.security.model.token.JwtTokenFactory.KEY_LENGTH;
@Component @Component
@RequiredArgsConstructor @RequiredArgsConstructor
public class DefaultJwtSettingsValidator implements JwtSettingsValidator { public class DefaultJwtSettingsValidator implements JwtSettingsValidator {
@ -59,8 +62,8 @@ public class DefaultJwtSettingsValidator implements JwtSettingsValidator {
if (Arrays.isNullOrEmpty(decodedKey)) { if (Arrays.isNullOrEmpty(decodedKey)) {
throw new DataValidationException("JWT token signing key should be non-empty after Base64 decoding!"); throw new DataValidationException("JWT token signing key should be non-empty after Base64 decoding!");
} }
if (decodedKey.length * Byte.SIZE < 256 && !JwtSettingsService.TOKEN_SIGNING_KEY_DEFAULT.equals(jwtSettings.getTokenSigningKey())) { if (decodedKey.length * Byte.SIZE < KEY_LENGTH && !isSigningKeyDefault(jwtSettings)) {
throw new DataValidationException("JWT token signing key should be a Base64 encoded string representing at least 256 bits of data!"); throw new DataValidationException("JWT token signing key should be a Base64 encoded string representing at least 512 bits of data!");
} }
System.arraycopy(decodedKey, 0, RandomUtils.nextBytes(decodedKey.length), 0, decodedKey.length); //secure memory System.arraycopy(decodedKey, 0, RandomUtils.nextBytes(decodedKey.length), 0, decodedKey.length); //secure memory

4
application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsService.java

@ -26,10 +26,6 @@ public interface JwtSettingsService {
JwtSettings reloadJwtSettings(); JwtSettings reloadJwtSettings();
void createRandomJwtSettings();
void saveLegacyYmlSettings();
JwtSettings saveJwtSettings(JwtSettings jwtSettings); JwtSettings saveJwtSettings(JwtSettings jwtSettings);
} }

72
application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java

@ -16,16 +16,19 @@
package org.thingsboard.server.service.security.model.token; package org.thingsboard.server.service.security.model.token;
import io.jsonwebtoken.Claims; import io.jsonwebtoken.Claims;
import io.jsonwebtoken.ClaimsBuilder;
import io.jsonwebtoken.ExpiredJwtException; import io.jsonwebtoken.ExpiredJwtException;
import io.jsonwebtoken.Jws; import io.jsonwebtoken.Jws;
import io.jsonwebtoken.JwtBuilder; import io.jsonwebtoken.JwtBuilder;
import io.jsonwebtoken.JwtParser;
import io.jsonwebtoken.Jwts; import io.jsonwebtoken.Jwts;
import io.jsonwebtoken.MalformedJwtException; import io.jsonwebtoken.MalformedJwtException;
import io.jsonwebtoken.SignatureAlgorithm;
import io.jsonwebtoken.SignatureException; import io.jsonwebtoken.SignatureException;
import io.jsonwebtoken.UnsupportedJwtException; import io.jsonwebtoken.UnsupportedJwtException;
import io.jsonwebtoken.security.Keys;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Lazy;
import org.springframework.security.authentication.BadCredentialsException; import org.springframework.security.authentication.BadCredentialsException;
import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.GrantedAuthority;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@ -41,7 +44,10 @@ import org.thingsboard.server.service.security.exception.JwtExpiredTokenExceptio
import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.model.UserPrincipal;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.util.Base64;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
@ -53,6 +59,8 @@ import java.util.stream.Collectors;
@Slf4j @Slf4j
public class JwtTokenFactory { public class JwtTokenFactory {
public static int KEY_LENGTH = Jwts.SIG.HS512.getKeyBitLength();
private static final String SCOPES = "scopes"; private static final String SCOPES = "scopes";
private static final String USER_ID = "userId"; private static final String USER_ID = "userId";
private static final String FIRST_NAME = "firstName"; private static final String FIRST_NAME = "firstName";
@ -63,8 +71,12 @@ public class JwtTokenFactory {
private static final String CUSTOMER_ID = "customerId"; private static final String CUSTOMER_ID = "customerId";
private static final String SESSION_ID = "sessionId"; private static final String SESSION_ID = "sessionId";
@Lazy
private final JwtSettingsService jwtSettingsService; private final JwtSettingsService jwtSettingsService;
private volatile JwtParser jwtParser;
private volatile SecretKey secretKey;
/** /**
* Factory method for issuing new JWT Tokens. * Factory method for issuing new JWT Tokens.
*/ */
@ -95,7 +107,7 @@ public class JwtTokenFactory {
public SecurityUser parseAccessJwtToken(String token) { public SecurityUser parseAccessJwtToken(String token) {
Jws<Claims> jwsClaims = parseTokenClaims(token); Jws<Claims> jwsClaims = parseTokenClaims(token);
Claims claims = jwsClaims.getBody(); Claims claims = jwsClaims.getPayload();
String subject = claims.getSubject(); String subject = claims.getSubject();
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
List<String> scopes = claims.get(SCOPES, List.class); List<String> scopes = claims.get(SCOPES, List.class);
@ -140,14 +152,14 @@ public class JwtTokenFactory {
String token = setUpToken(securityUser, Collections.singletonList(Authority.REFRESH_TOKEN.name()), jwtSettingsService.getJwtSettings().getRefreshTokenExpTime()) String token = setUpToken(securityUser, Collections.singletonList(Authority.REFRESH_TOKEN.name()), jwtSettingsService.getJwtSettings().getRefreshTokenExpTime())
.claim(IS_PUBLIC, principal.getType() == UserPrincipal.Type.PUBLIC_ID) .claim(IS_PUBLIC, principal.getType() == UserPrincipal.Type.PUBLIC_ID)
.setId(UUID.randomUUID().toString()).compact(); .id(UUID.randomUUID().toString()).compact();
return new AccessJwtToken(token); return new AccessJwtToken(token);
} }
public SecurityUser parseRefreshToken(String token) { public SecurityUser parseRefreshToken(String token) {
Jws<Claims> jwsClaims = parseTokenClaims(token); Jws<Claims> jwsClaims = parseTokenClaims(token);
Claims claims = jwsClaims.getBody(); Claims claims = jwsClaims.getPayload();
String subject = claims.getSubject(); String subject = claims.getSubject();
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
List<String> scopes = claims.get(SCOPES, List.class); List<String> scopes = claims.get(SCOPES, List.class);
@ -176,6 +188,11 @@ public class JwtTokenFactory {
return new AccessJwtToken(jwtBuilder.compact()); return new AccessJwtToken(jwtBuilder.compact());
} }
public void reload() {
getSecretKey(true);
getJwtParser(true);
}
private JwtBuilder setUpToken(SecurityUser securityUser, List<String> scopes, long expirationTime) { private JwtBuilder setUpToken(SecurityUser securityUser, List<String> scopes, long expirationTime) {
if (StringUtils.isBlank(securityUser.getEmail())) { if (StringUtils.isBlank(securityUser.getEmail())) {
throw new IllegalArgumentException("Cannot create JWT Token without username/email"); throw new IllegalArgumentException("Cannot create JWT Token without username/email");
@ -183,28 +200,27 @@ public class JwtTokenFactory {
UserPrincipal principal = securityUser.getUserPrincipal(); UserPrincipal principal = securityUser.getUserPrincipal();
Claims claims = Jwts.claims().setSubject(principal.getValue()); ClaimsBuilder claimsBuilder = Jwts.claims()
claims.put(USER_ID, securityUser.getId().getId().toString()); .subject(principal.getValue())
claims.put(SCOPES, scopes); .add(USER_ID, securityUser.getId().getId().toString())
.add(SCOPES, scopes);
if (securityUser.getSessionId() != null) { if (securityUser.getSessionId() != null) {
claims.put(SESSION_ID, securityUser.getSessionId()); claimsBuilder.add(SESSION_ID, securityUser.getSessionId());
} }
ZonedDateTime currentTime = ZonedDateTime.now(); ZonedDateTime currentTime = ZonedDateTime.now();
return Jwts.builder() return Jwts.builder()
.setClaims(claims) .claims(claimsBuilder.build())
.setIssuer(jwtSettingsService.getJwtSettings().getTokenIssuer()) .issuer(jwtSettingsService.getJwtSettings().getTokenIssuer())
.setIssuedAt(Date.from(currentTime.toInstant())) .issuedAt(Date.from(currentTime.toInstant()))
.setExpiration(Date.from(currentTime.plusSeconds(expirationTime).toInstant())) .expiration(Date.from(currentTime.plusSeconds(expirationTime).toInstant()))
.signWith(SignatureAlgorithm.HS512, jwtSettingsService.getJwtSettings().getTokenSigningKey()); .signWith(getSecretKey(false), Jwts.SIG.HS512);
} }
public Jws<Claims> parseTokenClaims(String token) { public Jws<Claims> parseTokenClaims(String token) {
try { try {
return Jwts.parser() return getJwtParser(false).parseSignedClaims(token);
.setSigningKey(jwtSettingsService.getJwtSettings().getTokenSigningKey())
.parseClaimsJws(token);
} catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException ex) { } catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException ex) {
log.debug("Invalid JWT Token", ex); log.debug("Invalid JWT Token", ex);
throw new BadCredentialsException("Invalid JWT token: ", ex); throw new BadCredentialsException("Invalid JWT token: ", ex);
@ -220,4 +236,28 @@ public class JwtTokenFactory {
return new JwtPair(accessToken.getToken(), refreshToken.getToken()); return new JwtPair(accessToken.getToken(), refreshToken.getToken());
} }
private SecretKey getSecretKey(boolean forceReload) {
if (secretKey == null || forceReload) {
synchronized (this) {
if (secretKey == null || forceReload) {
byte[] decodedToken = Base64.getDecoder().decode(jwtSettingsService.getJwtSettings().getTokenSigningKey());
secretKey = new SecretKeySpec(decodedToken, "HmacSHA512");
}
}
}
return secretKey;
}
private JwtParser getJwtParser(boolean forceReload) {
if (jwtParser == null || forceReload) {
synchronized (this) {
if (jwtParser == null || forceReload) {
jwtParser = Jwts.parser()
.verifyWith(Keys.hmacShaKeyFor(Base64.getDecoder().decode(jwtSettingsService.getJwtSettings().getTokenSigningKey())))
.build();
}
}
}
return jwtParser;
}
} }

6
application/src/main/java/org/thingsboard/server/service/security/model/token/OAuth2AppTokenFactory.java

@ -22,10 +22,12 @@ import io.jsonwebtoken.Jwts;
import io.jsonwebtoken.MalformedJwtException; import io.jsonwebtoken.MalformedJwtException;
import io.jsonwebtoken.SignatureException; import io.jsonwebtoken.SignatureException;
import io.jsonwebtoken.UnsupportedJwtException; import io.jsonwebtoken.UnsupportedJwtException;
import io.jsonwebtoken.security.Keys;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.StringUtils;
import java.util.Base64;
import java.util.Date; import java.util.Date;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -40,14 +42,14 @@ public class OAuth2AppTokenFactory {
public String validateTokenAndGetCallbackUrlScheme(String appPackage, String appToken, String appSecret) { public String validateTokenAndGetCallbackUrlScheme(String appPackage, String appToken, String appSecret) {
Jws<Claims> jwsClaims; Jws<Claims> jwsClaims;
try { try {
jwsClaims = Jwts.parser().setSigningKey(appSecret).parseClaimsJws(appToken); jwsClaims = Jwts.parser().verifyWith(Keys.hmacShaKeyFor(Base64.getDecoder().decode(appSecret))).build().parseSignedClaims(appToken);
} }
catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException | SignatureException ex) { catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException | SignatureException ex) {
throw new IllegalArgumentException("Invalid Application token: ", ex); throw new IllegalArgumentException("Invalid Application token: ", ex);
} catch (ExpiredJwtException expiredEx) { } catch (ExpiredJwtException expiredEx) {
throw new IllegalArgumentException("Application token expired", expiredEx); throw new IllegalArgumentException("Application token expired", expiredEx);
} }
Claims claims = jwsClaims.getBody(); Claims claims = jwsClaims.getPayload();
Date expiration = claims.getExpiration(); Date expiration = claims.getExpiration();
if (expiration == null) { if (expiration == null) {
throw new IllegalArgumentException("Application token must have expiration date"); throw new IllegalArgumentException("Application token must have expiration date");

28
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java

@ -108,34 +108,40 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi
return entity; return entity;
} }
@Override
public <E extends ExportableEntity<I>, I extends EntityId> E findDefaultEntityByTenantId(TenantId tenantId, EntityType entityType) {
ExportableEntityDao<I, E> dao = getExportableEntityDao(entityType);
if (dao == null) {
return null;
}
return dao.findDefaultEntityByTenantId(tenantId.getId());
}
@Override @Override
public <E extends ExportableEntity<I>, I extends EntityId> PageData<E> findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) { public <E extends ExportableEntity<I>, I extends EntityId> PageData<E> findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) {
ExportableEntityDao<I, E> dao = getExportableEntityDao(entityType); ExportableEntityDao<I, E> dao = getExportableEntityDao(entityType);
if (dao != null) { if (dao == null) {
return dao.findByTenantId(tenantId.getId(), pageLink); return PageData.emptyPageData();
} else {
return new PageData<>();
} }
return dao.findByTenantId(tenantId.getId(), pageLink);
} }
@Override @Override
public <I extends EntityId> PageData<I> findEntitiesIdsByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) { public <I extends EntityId> PageData<I> findEntitiesIdsByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) {
ExportableEntityDao<I, ?> dao = getExportableEntityDao(entityType); ExportableEntityDao<I, ?> dao = getExportableEntityDao(entityType);
if (dao != null) { if (dao == null) {
return dao.findIdsByTenantId(tenantId.getId(), pageLink); return PageData.emptyPageData();
} else {
return new PageData<>();
} }
return dao.findIdsByTenantId(tenantId.getId(), pageLink);
} }
@Override @Override
public <I extends EntityId> I getExternalIdByInternal(I internalId) { public <I extends EntityId> I getExternalIdByInternal(I internalId) {
ExportableEntityDao<I, ?> dao = getExportableEntityDao(internalId.getEntityType()); ExportableEntityDao<I, ?> dao = getExportableEntityDao(internalId.getEntityType());
if (dao != null) { if (dao == null) {
return dao.getExternalIdByInternal(internalId);
} else {
return null; return null;
} }
return dao.getExternalIdByInternal(internalId);
} }
private boolean belongsToTenant(HasId<? extends EntityId> entity, TenantId tenantId) { private boolean belongsToTenant(HasId<? extends EntityId> entity, TenantId tenantId) {

2
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java

@ -33,6 +33,8 @@ public interface ExportableEntitiesService {
<E extends ExportableEntity<I>, I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name); <E extends ExportableEntity<I>, I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name);
<E extends ExportableEntity<I>, I extends EntityId> E findDefaultEntityByTenantId(TenantId tenantId, EntityType entityType);
<E extends ExportableEntity<I>, I extends EntityId> PageData<E> findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); <E extends ExportableEntity<I>, I extends EntityId> PageData<E> findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink);
<I extends EntityId> PageData<I> findEntitiesIdsByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); <I extends EntityId> PageData<I> findEntitiesIdsByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink);

21
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java

@ -27,6 +27,7 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasDefaultOption;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
@ -73,7 +74,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
@Autowired @Autowired
@Lazy @Lazy
private ExportableEntitiesService exportableEntitiesService; private ExportableEntitiesService entitiesService;
@Autowired @Autowired
private RelationService relationService; private RelationService relationService;
@Autowired @Autowired
@ -274,22 +275,30 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) { protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) {
return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId())) return (E) Optional.ofNullable(entitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId()))
.or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId()))) .or(() -> Optional.ofNullable(entitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId())))
.or(() -> { .or(() -> {
if (ctx.isFindExistingByName()) { if (ctx.isFindExistingByName()) {
return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(ctx.getTenantId(), getEntityType(), entity.getName())); return Optional.ofNullable(entitiesService.findEntityByTenantIdAndName(ctx.getTenantId(), getEntityType(), entity.getName()));
} else { } else {
return Optional.empty(); return Optional.empty();
} }
}) })
.or(() -> {
if (entity instanceof HasDefaultOption hasDefaultOption) {
if (hasDefaultOption.isDefault()) {
return Optional.ofNullable(entitiesService.findDefaultEntityByTenantId(ctx.getTenantId(), getEntityType()));
}
}
return Optional.empty();
})
.orElse(null); .orElse(null);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private <ID extends EntityId> HasId<ID> findInternalEntity(TenantId tenantId, ID externalId) { private <ID extends EntityId> HasId<ID> findInternalEntity(TenantId tenantId, ID externalId) {
return (HasId<ID>) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) return (HasId<ID>) Optional.ofNullable(entitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId))
.or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) .or(() -> Optional.ofNullable(entitiesService.findEntityByTenantIdAndId(tenantId, externalId)))
.orElseThrow(() -> new MissingEntityException(externalId)); .orElseThrow(() -> new MissingEntityException(externalId));
} }

10
application/src/main/resources/thingsboard.yml

@ -164,6 +164,8 @@ mail:
oauth2: oauth2:
# Interval for checking refresh token expiration in seconds(by default, 1 day). # Interval for checking refresh token expiration in seconds(by default, 1 day).
refreshTokenCheckingInterval: "${REFRESH_TOKEN_EXPIRATION_CHECKING_INTERVAL:86400}" refreshTokenCheckingInterval: "${REFRESH_TOKEN_EXPIRATION_CHECKING_INTERVAL:86400}"
# Rate limits for sending mails per tenant. As example for 1000 per minute and 10000 per hour is "1000:60,10000:3600"
per_tenant_rate_limits: "${MAIL_PER_TENANT_RATE_LIMITS:}"
# Usage statistics parameters # Usage statistics parameters
usage: usage:
@ -505,6 +507,12 @@ cache:
assets: assets:
timeToLiveInMinutes: "${CACHE_SPECS_ASSETS_TTL:1440}" # Asset cache TTL timeToLiveInMinutes: "${CACHE_SPECS_ASSETS_TTL:1440}" # Asset cache TTL
maxSize: "${CACHE_SPECS_ASSETS_MAX_SIZE:10000}" # 0 means the cache is disabled maxSize: "${CACHE_SPECS_ASSETS_MAX_SIZE:10000}" # 0 means the cache is disabled
customers:
timeToLiveInMinutes: "${CACHE_SPECS_CUSTOMERS_TTL:1440}" # Customer cache TTL
maxSize: "${CACHE_SPECS_CUSTOMERS_MAX_SIZE:10000}" # 0 means the cache is disabled
users:
timeToLiveInMinutes: "${CACHE_SPECS_USERS_TTL:1440}" # User cache TTL
maxSize: "${CACHE_SPECS_USERS_MAX_SIZE:10000}" # 0 means the cache is disabled
entityViews: entityViews:
timeToLiveInMinutes: "${CACHE_SPECS_ENTITY_VIEWS_TTL:1440}" # Entity view cache TTL timeToLiveInMinutes: "${CACHE_SPECS_ENTITY_VIEWS_TTL:1440}" # Entity view cache TTL
maxSize: "${CACHE_SPECS_ENTITY_VIEWS_MAX_SIZE:10000}" # 0 means the cache is disabled maxSize: "${CACHE_SPECS_ENTITY_VIEWS_MAX_SIZE:10000}" # 0 means the cache is disabled
@ -1621,6 +1629,8 @@ queue:
partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}"
# Timeout for processing a message pack by Core microservices # Timeout for processing a message pack by Core microservices
pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}"
# Enable/disable a separate consumer per partition for Core queue
consumer-per-partition: "${TB_QUEUE_CORE_CONSUMER_PER_PARTITION:true}"
ota: ota:
# Default topic name for OTA updates # Default topic name for OTA updates
topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}" topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}"

17
application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@ -21,9 +21,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.ListeningExecutorService;
import io.jsonwebtoken.Claims; import io.jsonwebtoken.Claims;
import io.jsonwebtoken.Header; import io.jsonwebtoken.Jws;
import io.jsonwebtoken.Jwt;
import io.jsonwebtoken.Jwts;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.awaitility.Awaitility; import org.awaitility.Awaitility;
import org.hamcrest.Matcher; import org.hamcrest.Matcher;
@ -121,6 +119,7 @@ import org.thingsboard.server.queue.memory.InMemoryStorage;
import org.thingsboard.server.service.entitiy.tenant.profile.TbTenantProfileService; import org.thingsboard.server.service.entitiy.tenant.profile.TbTenantProfileService;
import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRequest; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRequest;
import org.thingsboard.server.service.security.auth.rest.LoginRequest; import org.thingsboard.server.service.security.auth.rest.LoginRequest;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import java.io.IOException; import java.io.IOException;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
@ -241,6 +240,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
@Autowired @Autowired
protected ClaimDevicesService claimDevicesService; protected ClaimDevicesService claimDevicesService;
@Autowired
private JwtTokenFactory jwtTokenFactory;
@SpyBean @SpyBean
protected MailService mailService; protected MailService mailService;
@ -561,13 +563,8 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
} }
protected void validateJwtToken(String token, String username) { protected void validateJwtToken(String token, String username) {
Assert.assertNotNull(token); Jws<Claims> jwsClaims = jwtTokenFactory.parseTokenClaims(token);
Assert.assertFalse(token.isEmpty()); Claims claims = jwsClaims.getPayload();
int i = token.lastIndexOf('.');
Assert.assertTrue(i > 0);
String withoutSignature = token.substring(0, i + 1);
Jwt<Header, Claims> jwsClaims = Jwts.parser().parseClaimsJwt(withoutSignature);
Claims claims = jwsClaims.getBody();
String subject = claims.getSubject(); String subject = claims.getSubject();
Assert.assertEquals(username, subject); Assert.assertEquals(username, subject);
} }

4
application/src/test/java/org/thingsboard/server/controller/AdminControllerTest.java

@ -42,7 +42,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
@Slf4j @Slf4j
@DaoSqlTest @DaoSqlTest
public class AdminControllerTest extends AbstractControllerTest { public class AdminControllerTest extends AbstractControllerTest {
final JwtSettings defaultJwtSettings = new JwtSettings(9000, 604800, "thingsboard.io", "thingsboardDefaultSigningKey"); final JwtSettings defaultJwtSettings = new JwtSettings(9000, 604800, "thingsboard.io", "QmlicmJkZk9tSzZPVFozcWY0Sm94UVhybmtBWXZ5YmZMOUZSZzZvcUFiOVhsb3VHUThhUWJGaXp3UHhtcGZ6Tw==");
@Test @Test
public void testFindAdminSettingsByKey() throws Exception { public void testFindAdminSettingsByKey() throws Exception {
@ -168,7 +168,7 @@ public class AdminControllerTest extends AbstractControllerTest {
assertThat(jwtSettings).isEqualTo(defaultJwtSettings); assertThat(jwtSettings).isEqualTo(defaultJwtSettings);
jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString( jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(
RandomStringUtils.randomAlphanumeric(256 / Byte.SIZE).getBytes(StandardCharsets.UTF_8))); RandomStringUtils.randomAlphanumeric(512 / Byte.SIZE).getBytes(StandardCharsets.UTF_8)));
doPost("/api/admin/jwtSettings", jwtSettings).andExpect(status().isOk()); doPost("/api/admin/jwtSettings", jwtSettings).andExpect(status().isOk());

7
application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java

@ -216,11 +216,12 @@ public class AuditLogControllerTest extends AbstractControllerTest {
@Test @Test
public void whenSavingAuditLogAndPartitionSaveErrorOccurred_thenSaveAuditLogAnyway() throws Exception { public void whenSavingAuditLogAndPartitionSaveErrorOccurred_thenSaveAuditLogAnyway() throws Exception {
// creating partition bigger than sql.audit_logs.partition_size // creating partition bigger than sql.audit_logs.partition_size
long entityTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-04-29T07:43:11Z").getTime(); long entityTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-29T07:43:11Z").getTime();
//the partition 7 days is overlapping default partition size 1 day, use in the far past to not affect other tests
partitioningRepository.createPartitionIfNotExists("audit_log", entityTs, TimeUnit.DAYS.toMillis(7)); partitioningRepository.createPartitionIfNotExists("audit_log", entityTs, TimeUnit.DAYS.toMillis(7));
List<Long> partitions = partitioningRepository.fetchPartitions("audit_log"); List<Long> partitions = partitioningRepository.fetchPartitions("audit_log");
log.warn("entityTs [{}], fetched partitions {}", entityTs, partitions); log.warn("entityTs [{}], fetched partitions {}", entityTs, partitions);
assertThat(partitions).contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-04-29T00:00:00Z").getTime()); assertThat(partitions).contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-28T00:00:00Z").getTime());
partitioningRepository.cleanupPartitionsCache("audit_log", entityTs, 0); partitioningRepository.cleanupPartitionsCache("audit_log", entityTs, 0);
assertDoesNotThrow(() -> { assertDoesNotThrow(() -> {
@ -228,7 +229,7 @@ public class AuditLogControllerTest extends AbstractControllerTest {
createAuditLog(ActionType.LOGIN, tenantAdminUserId, entityTs); createAuditLog(ActionType.LOGIN, tenantAdminUserId, entityTs);
}); });
assertThat(partitioningRepository.fetchPartitions("audit_log")) assertThat(partitioningRepository.fetchPartitions("audit_log"))
.contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-04-29T00:00:00Z").getTime());; .contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-28T00:00:00Z").getTime());;
} }
private AuditLog createAuditLog(ActionType actionType, EntityId entityId, long entityTs) { private AuditLog createAuditLog(ActionType actionType, EntityId entityId, long entityTs) {

6
application/src/test/java/org/thingsboard/server/controller/AuthControllerTest.java

@ -167,4 +167,10 @@ public class AuthControllerTest extends AbstractControllerTest {
.andExpect(jsonPath("$.message", .andExpect(jsonPath("$.message",
is("Password must be no more than 72 characters in length."))); is("Password must be no more than 72 characters in length.")));
} }
@Test
public void testGetPageWithoutRedirect() throws Exception {
doGet("/login").andExpect(status().isOk());
doGet("/home").andExpect(status().isOk());
}
} }

27
application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java

@ -352,7 +352,7 @@ public class CustomerControllerTest extends AbstractControllerTest {
} }
@Test @Test
public void testFindCustomersByTitle() throws Exception { public void testFindCustomersWithTitleAsTextSearch() throws Exception {
TenantId tenantId = savedTenant.getId(); TenantId tenantId = savedTenant.getId();
String title1 = "Customer title 1"; String title1 = "Customer title 1";
@ -424,6 +424,31 @@ public class CustomerControllerTest extends AbstractControllerTest {
Assert.assertEquals(0, pageData.getData().size()); Assert.assertEquals(0, pageData.getData().size());
} }
@Test
public void testFindCustomerByTitle() throws Exception {
Customer customer = new Customer();
customer.setTitle("My customer");
Mockito.reset(tbClusterService, auditLogService);
Customer savedCustomer = doPost("/api/customer", customer, Customer.class);
testNotifyEntityAllOneTime(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), savedCustomer.getTenantId(),
new CustomerId(CustomerId.NULL_UUID), tenantAdmin.getId(), tenantAdmin.getEmail(),
ActionType.ADDED);
Assert.assertNotNull(savedCustomer);
Assert.assertNotNull(savedCustomer.getId());
Assert.assertTrue(savedCustomer.getCreatedTime() > 0);
Assert.assertEquals(customer.getTitle(), savedCustomer.getTitle());
Customer foundCustomer = doGet("/api/tenant/customers?customerTitle=" + savedCustomer.getTitle(), Customer.class);
Assert.assertEquals(foundCustomer, savedCustomer);
doDelete("/api/customer/" + savedCustomer.getId().getId().toString())
.andExpect(status().isOk());
}
@Test @Test
public void testDeleteCustomerWithDeleteRelationsOk() throws Exception { public void testDeleteCustomerWithDeleteRelationsOk() throws Exception {
CustomerId customerId = createCustomer("Customer for Test WithRelationsOk").getId(); CustomerId customerId = createCustomer("Customer for Test WithRelationsOk").getId();

38
application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java

@ -84,6 +84,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never; import static org.mockito.Mockito.never;
@ -1541,17 +1542,24 @@ public class DeviceControllerTest extends AbstractControllerTest {
Device savedDevice = doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class); Device savedDevice = doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class);
//check server attribute value //check server attribute value
List<Map<String, Object>> values = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
"/values/attributes/SERVER_SCOPE", new TypeReference<>() { Map<String, Object> actualAttribute = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<List<Map<String, Object>>>() {}).stream()
.filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(attributeValue, actualAttribute.get("value"));
}); });
Map<String, Object> serverAttribute = values.stream().filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(attributeValue, serverAttribute.get("value"));
//update server attribute value //update server attribute value
String newAttributeValue = "testValue2"; String newAttributeValue = "testValue2";
JsonNode content = JacksonUtil.toJsonNode("{\"DATA\": \"" + newAttributeValue + "\"}"); JsonNode content = JacksonUtil.toJsonNode("{\"DATA\": \"" + newAttributeValue + "\"}");
doPost("/api/plugins/telemetry/" + EntityType.DEVICE.name() + "/" + savedDevice.getUuidId() + "/SERVER_SCOPE", content) doPost("/api/plugins/telemetry/" + EntityType.DEVICE.name() + "/" + savedDevice.getUuidId() + "/SERVER_SCOPE", content)
.andExpect(status().isOk()); .andExpect(status().isOk());
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
Map<String, Object> actualAttribute = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<List<Map<String, Object>>>() {}).stream()
.filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(newAttributeValue, actualAttribute.get("value"));
});
//reimport devices //reimport devices
String deviceName2 = "secondDevice"; String deviceName2 = "secondDevice";
@ -1563,19 +1571,19 @@ public class DeviceControllerTest extends AbstractControllerTest {
doPostWithTypedResponse("/api/device/bulk_import", request, new TypeReference<>() {}); doPostWithTypedResponse("/api/device/bulk_import", request, new TypeReference<>() {});
Device savedDevice2 = doGet("/api/tenant/devices?deviceName=" + deviceName2, Device.class); Device savedDevice2 = doGet("/api/tenant/devices?deviceName=" + deviceName2, Device.class);
//check attribute value was not changed after reimport
List<Map<String, Object>> values2 = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<>() {
});
Map<String, Object> retrievedServerAttribute2 = values2.stream().filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(newAttributeValue, retrievedServerAttribute2.get("value"));
//check attribute for second device //check attribute for second device
List<Map<String, Object>> values3 = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice2.getId() + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
"/values/attributes/SERVER_SCOPE", new TypeReference<>() { Map<String, Object> actualAttribute = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice2.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<List<Map<String, Object>>>() {}).stream()
.filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(attributeValue2, actualAttribute.get("value"));
}); });
Map<String, Object> retrievedServerAttribute3 = values3.stream().filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(attributeValue2, retrievedServerAttribute3.get("value")); //check attribute value was not changed after reimport
Map<String, Object> actualAttribute = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<List<Map<String, Object>>>() {}).stream()
.filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(newAttributeValue, actualAttribute.get("value"));
} }
private Device createDevice(String name) { private Device createDevice(String name) {

13
application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java

@ -275,10 +275,9 @@ public class EntityViewControllerTest extends AbstractControllerTest {
} }
@Test @Test
public void testAssignAndUnAssignedEntityViewToCustomer() throws Exception { public void testAssignAndUnassignEntityViewToCustomer() throws Exception {
EntityView view = getNewSavedEntityView("Test entity view"); EntityView view = getNewSavedEntityView("Test entity view");
Customer savedCustomer = doPost("/api/customer", getNewCustomer("My customer"), Customer.class); Customer savedCustomer = doPost("/api/customer", getNewCustomer("My customer"), Customer.class);
view.setCustomerId(savedCustomer.getId());
Mockito.reset(tbClusterService, auditLogService); Mockito.reset(tbClusterService, auditLogService);
@ -304,17 +303,17 @@ public class EntityViewControllerTest extends AbstractControllerTest {
ActionType.ASSIGNED_TO_CUSTOMER, ActionType.UPDATED, ActionType.ASSIGNED_TO_CUSTOMER, ActionType.UPDATED,
foundView.getId().getId().toString(), foundView.getCustomerId().getId().toString(), savedCustomer.getTitle()); foundView.getId().getId().toString(), foundView.getCustomerId().getId().toString(), savedCustomer.getTitle());
EntityView unAssignedView = doDelete("/api/customer/entityView/" + savedView.getId().getId().toString(), EntityView.class); EntityView unassignedView = doDelete("/api/customer/entityView/" + savedView.getId().getId().toString(), EntityView.class);
assertEquals(ModelConstants.NULL_UUID, unAssignedView.getCustomerId().getId()); assertEquals(ModelConstants.NULL_UUID, unassignedView.getCustomerId().getId());
foundView = doGet("/api/entityView/" + savedView.getId().getId().toString(), EntityView.class); foundView = doGet("/api/entityView/" + savedView.getId().getId().toString(), EntityView.class);
assertEquals(ModelConstants.NULL_UUID, foundView.getCustomerId().getId()); assertEquals(ModelConstants.NULL_UUID, foundView.getCustomerId().getId());
testBroadcastEntityStateChangeEventTime(foundView.getId(), foundView.getTenantId(), 1); testBroadcastEntityStateChangeEventTime(foundView.getId(), foundView.getTenantId(), 1);
testNotifyAssignUnassignEntityAllOneTime(unAssignedView, savedView.getId(), savedView.getId(), testNotifyAssignUnassignEntityAllOneTime(unassignedView, unassignedView.getId(), unassignedView.getId(),
tenantId, savedView.getCustomerId(), tenantAdminUserId, TENANT_ADMIN_EMAIL, tenantId, savedCustomer.getId(), tenantAdminUserId, TENANT_ADMIN_EMAIL,
ActionType.UNASSIGNED_FROM_CUSTOMER, ActionType.UPDATED, ActionType.UNASSIGNED_FROM_CUSTOMER, ActionType.UPDATED,
assignedView.getId().getId().toString(), savedView.getCustomerId().getId().toString(), savedCustomer.getTitle()); assignedView.getId().getId().toString(), savedCustomer.getId().getId().toString(), savedCustomer.getTitle());
} }
@Test @Test

2
application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java

@ -279,7 +279,7 @@ public class UserControllerTest extends AbstractControllerTest {
user.setTenantId(tenantId); user.setTenantId(tenantId);
user.setEmail(TENANT_ADMIN_EMAIL); user.setEmail(TENANT_ADMIN_EMAIL);
String msgError = "User with email '" + TENANT_ADMIN_EMAIL + "' already present in database"; String msgError = "User with email '" + TENANT_ADMIN_EMAIL + "' already present in database!";
doPost("/api/user", user) doPost("/api/user", user)
.andExpect(status().isBadRequest()) .andExpect(status().isBadRequest())
.andExpect(statusReason(containsString(msgError))); .andExpect(statusReason(containsString(msgError)));

62
application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java

@ -27,6 +27,8 @@ import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner; import org.mockito.junit.MockitoJUnitRunner;
import org.testcontainers.shaded.org.apache.commons.lang3.RandomUtils; import org.testcontainers.shaded.org.apache.commons.lang3.RandomUtils;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.QueueId;
@ -65,6 +67,9 @@ import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier; import java.util.function.Supplier;
@ -86,7 +91,6 @@ import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy; import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times; import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@ -112,6 +116,9 @@ public class TbRuleEngineQueueConsumerManagerTest {
@Mock @Mock
private TbQueueAdmin queueAdmin; private TbQueueAdmin queueAdmin;
private TbRuleEngineConsumerContext ruleEngineConsumerContext; private TbRuleEngineConsumerContext ruleEngineConsumerContext;
private ExecutorService consumersExecutor;
private ScheduledExecutorService scheduler;
private ExecutorService mgmtExecutor;
private TbRuleEngineQueueConsumerManager consumerManager; private TbRuleEngineQueueConsumerManager consumerManager;
private Queue queue; private Queue queue;
@ -141,10 +148,10 @@ public class TbRuleEngineQueueConsumerManagerTest {
}).when(actorContext).tell(any()); }).when(actorContext).tell(any());
ruleEngineMsgProducer = mock(TbQueueProducer.class); ruleEngineMsgProducer = mock(TbQueueProducer.class);
when(producerProvider.getRuleEngineMsgProducer()).thenReturn(ruleEngineMsgProducer); when(producerProvider.getRuleEngineMsgProducer()).thenReturn(ruleEngineMsgProducer);
ruleEngineConsumerContext.setMgmtThreadPoolSize(2); consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer"));
mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(3, "tb-rule-engine-mgmt");
scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler"));
ruleEngineConsumerContext.setTopicDeletionDelayInSec(5); ruleEngineConsumerContext.setTopicDeletionDelayInSec(5);
ruleEngineConsumerContext.init();
ruleEngineConsumerContext.setReady(false);
queue = new Queue(); queue = new Queue();
queue.setName("Test"); queue.setName("Test");
@ -174,14 +181,23 @@ public class TbRuleEngineQueueConsumerManagerTest {
}).when(queueFactory).createToRuleEngineMsgConsumer(any()); }).when(queueFactory).createToRuleEngineMsgConsumer(any());
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queue); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queue);
consumerManager = new TbRuleEngineQueueConsumerManager(ruleEngineConsumerContext, queueKey); consumerManager = TbRuleEngineQueueConsumerManager.create()
.ctx(ruleEngineConsumerContext)
.queueKey(queueKey)
.consumerExecutor(consumersExecutor)
.scheduler(scheduler)
.taskExecutor(mgmtExecutor)
.build();
} }
@After @After
public void afterEach() { public void afterEach() {
consumerManager.stop(); consumerManager.stop();
consumerManager.awaitStop(); consumerManager.awaitStop();
ruleEngineConsumerContext.stop();
consumersExecutor.shutdownNow();
scheduler.shutdownNow();
mgmtExecutor.shutdownNow();
if (generateQueueMsgs) { if (generateQueueMsgs) {
await().atMost(10, TimeUnit.SECONDS) await().atMost(10, TimeUnit.SECONDS)
@ -199,14 +215,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
Set<TopicPartitionInfo> partitions = createTpis(2, 3, 4); Set<TopicPartitionInfo> partitions = createTpis(2, 3, 4);
consumerManager.update(partitions); consumerManager.update(partitions);
partitions = createTpis(3, 4, 5);
consumerManager.update(partitions);
partitions = createTpis(1, 2, 3);
consumerManager.update(partitions);
// simulated multiple partition change events before consumer is ready; only latest partitions should be processed
verifyNoInteractions(queueFactory);
ruleEngineConsumerContext.setReady(true);
await().atMost(2, TimeUnit.SECONDS) await().atMost(2, TimeUnit.SECONDS)
.until(() -> consumers.size() == 3); .until(() -> consumers.size() == 3);
for (TopicPartitionInfo partition : partitions) { for (TopicPartitionInfo partition : partitions) {
@ -222,14 +231,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
Set<TopicPartitionInfo> partitions = createTpis(2, 3, 4); Set<TopicPartitionInfo> partitions = createTpis(2, 3, 4);
consumerManager.update(partitions); consumerManager.update(partitions);
partitions = createTpis(3, 4, 5);
consumerManager.update(partitions);
partitions = createTpis(1, 2, 3);
consumerManager.update(partitions);
verifyNoInteractions(queueFactory);
ruleEngineConsumerContext.setReady(true);
await().atMost(2, TimeUnit.SECONDS) await().atMost(2, TimeUnit.SECONDS)
.until(() -> consumers.size() == 1); .until(() -> consumers.size() == 1);
TestConsumer consumer = getConsumer(); TestConsumer consumer = getConsumer();
@ -240,7 +242,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testPartitionsUpdate_singleConsumer() { public void testPartitionsUpdate_singleConsumer() {
queue.setConsumerPerPartition(false); queue.setConsumerPerPartition(false);
consumerManager.init(queue); consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = Collections.emptySet(); Set<TopicPartitionInfo> partitions = Collections.emptySet();
consumerManager.update(partitions); consumerManager.update(partitions);
@ -273,7 +274,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testPartitionsUpdate_consumerPerPartition() { public void testPartitionsUpdate_consumerPerPartition() {
queue.setConsumerPerPartition(true); queue.setConsumerPerPartition(true);
consumerManager.init(queue); consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
consumerManager.update(Collections.emptySet()); consumerManager.update(Collections.emptySet());
verify(queueFactory, after(1000).never()).createToRuleEngineMsgConsumer(any()); verify(queueFactory, after(1000).never()).createToRuleEngineMsgConsumer(any());
@ -316,7 +316,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testConfigUpdate_singleConsumer() { public void testConfigUpdate_singleConsumer() {
queue.setConsumerPerPartition(false); queue.setConsumerPerPartition(false);
consumerManager.init(queue); consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3); Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3);
consumerManager.update(partitions); consumerManager.update(partitions);
TestConsumer consumer = getConsumer(); TestConsumer consumer = getConsumer();
@ -342,7 +341,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testConfigUpdate_consumerPerPartition() { public void testConfigUpdate_consumerPerPartition() {
queue.setConsumerPerPartition(true); queue.setConsumerPerPartition(true);
consumerManager.init(queue); consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3); Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3);
consumerManager.update(partitions); consumerManager.update(partitions);
TestConsumer consumer1 = getConsumer(1); TestConsumer consumer1 = getConsumer(1);
@ -375,7 +373,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testConfigUpdate_fromSingleToConsumerPerPartition() { public void testConfigUpdate_fromSingleToConsumerPerPartition() {
queue.setConsumerPerPartition(false); queue.setConsumerPerPartition(false);
consumerManager.init(queue); consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3); Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3);
consumerManager.update(partitions); consumerManager.update(partitions);
TestConsumer consumer = getConsumer(); TestConsumer consumer = getConsumer();
@ -395,7 +392,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testConfigUpdate_fromConsumerPerPartitionToSingle() { public void testConfigUpdate_fromConsumerPerPartitionToSingle() {
queue.setConsumerPerPartition(true); queue.setConsumerPerPartition(true);
consumerManager.init(queue); consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3); Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3);
consumerManager.update(partitions); consumerManager.update(partitions);
TestConsumer consumer1 = getConsumer(1); TestConsumer consumer1 = getConsumer(1);
@ -419,7 +415,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testStop() { public void testStop() {
queue.setConsumerPerPartition(true); queue.setConsumerPerPartition(true);
consumerManager.init(queue); consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
consumerManager.update(createTpis(1)); consumerManager.update(createTpis(1));
TestConsumer consumer = getConsumer(1); TestConsumer consumer = getConsumer(1);
verifySubscribedAndLaunched(consumer, 1); verifySubscribedAndLaunched(consumer, 1);
@ -437,7 +432,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testDelete_consumerPerPartition() { public void testDelete_consumerPerPartition() {
queue.setConsumerPerPartition(true); queue.setConsumerPerPartition(true);
consumerManager.init(queue); consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2); Set<TopicPartitionInfo> partitions = createTpis(1, 2);
consumerManager.update(partitions); consumerManager.update(partitions);
TestConsumer consumer1 = getConsumer(1); TestConsumer consumer1 = getConsumer(1);
@ -459,7 +453,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
int msgCount = totalConsumedMsgs.get(); int msgCount = totalConsumedMsgs.get();
await().atLeast(2, TimeUnit.SECONDS) // based on topicDeletionDelayInSec(5) = 5 - ( 3 seconds the code may execute starting consumerManager.delete() call) await().atLeast(2, TimeUnit.SECONDS) // based on topicDeletionDelayInSec(5) = 5 - ( 3 seconds the code may execute starting consumerManager.delete() call)
.atMost(7, TimeUnit.SECONDS) .atMost(20, TimeUnit.SECONDS)
.untilAsserted(() -> { .untilAsserted(() -> {
partitions.stream() partitions.stream()
.map(TopicPartitionInfo::getFullTopicName) .map(TopicPartitionInfo::getFullTopicName)
@ -481,7 +475,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testDelete_singleConsumer() { public void testDelete_singleConsumer() {
queue.setConsumerPerPartition(false); queue.setConsumerPerPartition(false);
consumerManager.init(queue); consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2); Set<TopicPartitionInfo> partitions = createTpis(1, 2);
consumerManager.update(partitions); consumerManager.update(partitions);
TestConsumer consumer = getConsumer(); TestConsumer consumer = getConsumer();
@ -499,7 +492,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
int msgCount = totalConsumedMsgs.get(); int msgCount = totalConsumedMsgs.get();
await().atLeast(2, TimeUnit.SECONDS) // based on topicDeletionDelayInSec(5) = 5 - ( 3 seconds the code may execute starting consumerManager.delete() call) await().atLeast(2, TimeUnit.SECONDS) // based on topicDeletionDelayInSec(5) = 5 - ( 3 seconds the code may execute starting consumerManager.delete() call)
.atMost(7, TimeUnit.SECONDS) .atMost(20, TimeUnit.SECONDS)
.untilAsserted(() -> { .untilAsserted(() -> {
partitions.stream() partitions.stream()
.map(TopicPartitionInfo::getFullTopicName) .map(TopicPartitionInfo::getFullTopicName)
@ -520,10 +513,9 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testManyDifferentUpdates() throws Exception { public void testManyDifferentUpdates() throws Exception {
queue.setConsumerPerPartition(RandomUtils.nextBoolean()); queue.setConsumerPerPartition(RandomUtils.nextBoolean());
consumerManager.init(queue); consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Supplier<Queue> queueConfigUpdater = () -> { Supplier<Queue> queueConfigUpdater = () -> {
Queue oldConfig = consumerManager.getQueue(); Queue oldConfig = consumerManager.getConfig();
Queue newConfig = JacksonUtil.clone(oldConfig); Queue newConfig = JacksonUtil.clone(oldConfig);
newConfig.setConsumerPerPartition(RandomUtils.nextBoolean()); newConfig.setConsumerPerPartition(RandomUtils.nextBoolean());
newConfig.setPollInterval(RandomUtils.nextInt(100, 501)); newConfig.setPollInterval(RandomUtils.nextInt(100, 501));
@ -571,7 +563,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
Set<TopicPartitionInfo> expectedPartitions = latestPartitions; Set<TopicPartitionInfo> expectedPartitions = latestPartitions;
await().atMost(5, TimeUnit.SECONDS) await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> { .untilAsserted(() -> {
assertThat(consumerManager.getQueue()).isEqualTo(expectedConfig); assertThat(consumerManager.getConfig()).isEqualTo(expectedConfig);
assertThat(consumerManager.getPartitions()).isEqualTo(expectedPartitions); assertThat(consumerManager.getPartitions()).isEqualTo(expectedPartitions);
}); });
@ -640,7 +632,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
} }
private void verifyMsgProcessed(TbMsg tbMsg) { private void verifyMsgProcessed(TbMsg tbMsg) {
await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> { await().atMost(15, TimeUnit.SECONDS).untilAsserted(() -> {
verify(actorContext, atLeastOnce()).tell(argThat(msg -> { verify(actorContext, atLeastOnce()).tell(argThat(msg -> {
return ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(tbMsg.getId()); return ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(tbMsg.getId());
})); }));

35
application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java

@ -16,16 +16,14 @@
package org.thingsboard.server.service.security.auth; package org.thingsboard.server.service.security.auth;
import io.jsonwebtoken.Claims; import io.jsonwebtoken.Claims;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.notification.NotificationDeliveryMethod;
import org.thingsboard.server.common.data.notification.targets.platform.SystemAdministratorsFilter;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.common.data.security.model.JwtToken; import org.thingsboard.server.common.data.security.model.JwtToken;
@ -38,6 +36,8 @@ import org.thingsboard.server.service.security.model.UserPrincipal;
import org.thingsboard.server.service.security.model.token.AccessJwtToken; import org.thingsboard.server.service.security.model.token.AccessJwtToken;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Calendar; import java.util.Calendar;
import java.util.Date; import java.util.Date;
import java.util.Optional; import java.util.Optional;
@ -45,19 +45,13 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class JwtTokenFactoryTest { public class JwtTokenFactoryTest {
private JwtTokenFactory tokenFactory; private JwtTokenFactory tokenFactory;
private AdminSettingsService adminSettingsService; private AdminSettingsService adminSettingsService;
private NotificationCenter notificationCenter;
private JwtSettingsService jwtSettingsService; private JwtSettingsService jwtSettingsService;
private JwtSettings jwtSettings; private JwtSettings jwtSettings;
@ -66,12 +60,11 @@ public class JwtTokenFactoryTest {
public void beforeEach() { public void beforeEach() {
jwtSettings = new JwtSettings(); jwtSettings = new JwtSettings();
jwtSettings.setTokenIssuer("tb"); jwtSettings.setTokenIssuer("tb");
jwtSettings.setTokenSigningKey("abewafaf"); jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8)));
jwtSettings.setTokenExpirationTime((int) TimeUnit.HOURS.toSeconds(2)); jwtSettings.setTokenExpirationTime((int) TimeUnit.HOURS.toSeconds(2));
jwtSettings.setRefreshTokenExpTime((int) TimeUnit.DAYS.toSeconds(7)); jwtSettings.setRefreshTokenExpTime((int) TimeUnit.DAYS.toSeconds(7));
adminSettingsService = mock(AdminSettingsService.class); adminSettingsService = mock(AdminSettingsService.class);
notificationCenter = mock(NotificationCenter.class);
jwtSettingsService = mockJwtSettingsService(); jwtSettingsService = mockJwtSettingsService();
mockJwtSettings(jwtSettings); mockJwtSettings(jwtSettings);
@ -169,21 +162,6 @@ public class JwtTokenFactoryTest {
}); });
} }
@Test
public void testJwtSigningKeyIssueNotification() {
JwtSettings badJwtSettings = jwtSettings;
badJwtSettings.setTokenSigningKey(JwtSettingsService.TOKEN_SIGNING_KEY_DEFAULT);
mockJwtSettings(badJwtSettings);
jwtSettingsService = mockJwtSettingsService();
for (int i = 0; i < 5; i++) { // to check if notification is not sent twice
jwtSettingsService.getJwtSettings();
}
verify(notificationCenter, times(1)).sendGeneralWebNotification(eq(TenantId.SYS_TENANT_ID),
isA(SystemAdministratorsFilter.class), argThat(template -> template.getConfiguration().getDeliveryMethodsTemplates().get(NotificationDeliveryMethod.WEB)
.getBody().contains("The platform is configured to use default JWT Signing Key")));
}
private void mockJwtSettings(JwtSettings settings) { private void mockJwtSettings(JwtSettings settings) {
AdminSettings adminJwtSettings = new AdminSettings(); AdminSettings adminJwtSettings = new AdminSettings();
adminJwtSettings.setJsonValue(JacksonUtil.valueToTree(settings)); adminJwtSettings.setJsonValue(JacksonUtil.valueToTree(settings));
@ -192,12 +170,11 @@ public class JwtTokenFactoryTest {
} }
private DefaultJwtSettingsService mockJwtSettingsService() { private DefaultJwtSettingsService mockJwtSettingsService() {
return new DefaultJwtSettingsService(adminSettingsService, Optional.empty(), return new DefaultJwtSettingsService(adminSettingsService, Optional.empty(), new DefaultJwtSettingsValidator(), Optional.empty());
Optional.of(notificationCenter), new DefaultJwtSettingsValidator());
} }
private void checkExpirationTime(JwtToken jwtToken, int tokenLifetime) { private void checkExpirationTime(JwtToken jwtToken, int tokenLifetime) {
Claims claims = tokenFactory.parseTokenClaims(jwtToken.getToken()).getBody(); Claims claims = tokenFactory.parseTokenClaims(jwtToken.getToken()).getPayload();
assertThat(claims.getExpiration()).matches(actualExpirationTime -> { assertThat(claims.getExpiration()).matches(actualExpirationTime -> {
Calendar expirationTime = Calendar.getInstance(); Calendar expirationTime = Calendar.getInstance();
expirationTime.setTime(new Date()); expirationTime.setTime(new Date());

17
application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java

@ -532,20 +532,19 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest {
@Test @Test
public void testExportImportDeviceProfile_betweenTenants_findExistingByName() throws Exception { public void testExportImportDefaultDeviceProfile_betweenTenants_findExisting() throws Exception {
DeviceProfile defaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId1); DeviceProfile defaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId1);
defaultDeviceProfile.setName("non-default-name");
deviceProfileService.saveDeviceProfile(defaultDeviceProfile);
EntityExportData<DeviceProfile> deviceProfileExportData = exportEntity(tenantAdmin1, defaultDeviceProfile.getId()); EntityExportData<DeviceProfile> deviceProfileExportData = exportEntity(tenantAdmin1, defaultDeviceProfile.getId());
assertThatThrownBy(() -> {
importEntity(tenantAdmin2, deviceProfileExportData, EntityImportSettings.builder()
.findExistingByName(false)
.build());
}).hasMessageContaining("default device profile is present");
importEntity(tenantAdmin2, deviceProfileExportData, EntityImportSettings.builder() importEntity(tenantAdmin2, deviceProfileExportData, EntityImportSettings.builder()
.findExistingByName(true) .findExistingByName(false)
.build()); .build());
checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, deviceProfileService.findDefaultDeviceProfile(tenantId2)); DeviceProfile importedDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId2);
assertThat(importedDeviceProfile.isDefault()).isTrue();
assertThat(importedDeviceProfile.getName()).isEqualTo(defaultDeviceProfile.getName());
checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, importedDeviceProfile);
} }

5
common/actor/pom.xml

@ -65,11 +65,6 @@
<artifactId>spring-boot-starter-test</artifactId> <artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.awaitility</groupId> <groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId> <artifactId>awaitility</artifactId>

69
common/actor/src/test/java/org/thingsboard/server/actors/ActorSystemTest.java

@ -16,12 +16,11 @@
package org.thingsboard.server.actors; package org.thingsboard.server.actors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.junit.After; import org.awaitility.Awaitility;
import org.junit.Assert; import org.junit.jupiter.api.AfterEach;
import org.junit.Before; import org.junit.jupiter.api.Assertions;
import org.junit.Test; import org.junit.jupiter.api.BeforeEach;
import org.junit.runner.RunWith; import org.junit.jupiter.api.Test;
import org.mockito.junit.MockitoJUnitRunner;
import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceId;
@ -37,29 +36,31 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import static org.assertj.core.api.Assertions.assertThat;
@Slf4j @Slf4j
@RunWith(MockitoJUnitRunner.class)
public class ActorSystemTest { public class ActorSystemTest {
public static final String ROOT_DISPATCHER = "root-dispatcher"; public static final String ROOT_DISPATCHER = "root-dispatcher";
private static final int _100K = 100 * 1024; private static final int _100K = 100 * 1024;
public static final int TIMEOUT_AWAIT_MAX_SEC = 100; public static final int TIMEOUT_AWAIT_MAX_SEC = 30;
private volatile TbActorSystem actorSystem; private volatile TbActorSystem actorSystem;
private volatile ExecutorService submitPool; private volatile ExecutorService submitPool;
private ExecutorService executor; private ExecutorService executor;
private int parallelism; private int parallelism;
@Before @BeforeEach
public void initActorSystem() { public void initActorSystem() {
int cores = Runtime.getRuntime().availableProcessors(); int cores = Runtime.getRuntime().availableProcessors();
parallelism = Math.max(2, cores / 2); parallelism = Math.max(2, cores / 2);
log.debug("parallelism {}", parallelism);
TbActorSystemSettings settings = new TbActorSystemSettings(5, parallelism, 42); TbActorSystemSettings settings = new TbActorSystemSettings(5, parallelism, 42);
actorSystem = new DefaultTbActorSystem(settings); actorSystem = new DefaultTbActorSystem(settings);
submitPool = Executors.newFixedThreadPool(parallelism, ThingsBoardThreadFactory.forName(getClass().getSimpleName() + "-submit-test-scope")); //order guaranteed submitPool = Executors.newFixedThreadPool(parallelism, ThingsBoardThreadFactory.forName(getClass().getSimpleName() + "-submit-test-scope")); //order guaranteed
} }
@After @AfterEach
public void shutdownActorSystem() { public void shutdownActorSystem() {
actorSystem.stop(); actorSystem.stop();
submitPool.shutdownNow(); submitPool.shutdownNow();
@ -126,8 +127,8 @@ public class ActorSystemTest {
actorId2.tell(new IntTbActorMsg(42)); actorId2.tell(new IntTbActorMsg(42));
actorSystem.stop(actorId1); actorSystem.stop(actorId1);
Assert.assertTrue(testCtx2.getLatch().await(1, TimeUnit.SECONDS)); Assertions.assertTrue(testCtx2.getLatch().await(1, TimeUnit.SECONDS));
Assert.assertFalse(testCtx1.getLatch().await(1, TimeUnit.SECONDS)); Assertions.assertFalse(testCtx1.getLatch().await(1, TimeUnit.SECONDS));
} }
@Test @Test
@ -136,25 +137,39 @@ public class ActorSystemTest {
actorSystem.createDispatcher(ROOT_DISPATCHER, executor); actorSystem.createDispatcher(ROOT_DISPATCHER, executor);
ActorTestCtx testCtx1 = getActorTestCtx(1); ActorTestCtx testCtx1 = getActorTestCtx(1);
ActorTestCtx testCtx2 = getActorTestCtx(1); ActorTestCtx testCtx2 = getActorTestCtx(1);
assertThat(testCtx1.getLatch().getCount()).as("testCtx1 latch initial state").isEqualTo(1);
assertThat(testCtx2.getLatch().getCount()).as("testCtx2 latch initial state").isEqualTo(1);
TbActorId actorId = new TbEntityActorId(new DeviceId(UUID.randomUUID())); TbActorId actorId = new TbEntityActorId(new DeviceId(UUID.randomUUID()));
final CountDownLatch initLatch = new CountDownLatch(1); final CountDownLatch initLatch = new CountDownLatch(1);
final CountDownLatch actorsReadyLatch = new CountDownLatch(2); final CountDownLatch actorsReadyLatch = new CountDownLatch(2);
submitPool.submit(() -> { submitPool.submit(() -> {
log.info("submit 1");
actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx1, initLatch)); actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx1, initLatch));
actorsReadyLatch.countDown(); actorsReadyLatch.countDown();
log.info("done 1");
}); });
submitPool.submit(() -> { submitPool.submit(() -> {
log.info("submit 2");
actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx2, initLatch)); actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx2, initLatch));
actorsReadyLatch.countDown(); actorsReadyLatch.countDown();
log.info("done 2");
}); });
initLatch.countDown(); //replacement for Thread.wait(500) in the SlowCreateActorCreator initLatch.countDown(); //replacement for Thread.wait(500) in the SlowCreateActorCreator
Assert.assertTrue(actorsReadyLatch.await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS)); Assertions.assertTrue(actorsReadyLatch.await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS));
log.info("actorsReadyLatch ok");
actorSystem.tell(actorId, new IntTbActorMsg(42)); actorSystem.tell(actorId, new IntTbActorMsg(42));
Assert.assertTrue(testCtx1.getLatch().await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS)); //only one of two contexts are initialized. no matter Ctx1 or Ctx2
Assert.assertFalse(testCtx2.getLatch().await(1, TimeUnit.SECONDS)); Awaitility.await("one of two actors latch zeroed").atMost(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS)
.until(() -> testCtx1.getLatch().getCount() + testCtx2.getLatch().getCount() == 1);
Thread.yield();
if (testCtx1.getLatch().getCount() == 0) {
assertThat(testCtx2.getLatch().await(100, TimeUnit.MILLISECONDS)).as("testCtx2 never latched").isFalse();
} else {
assertThat(testCtx1.getLatch().await(100, TimeUnit.MILLISECONDS)).as("testCtx1 never latched").isFalse();
}
} }
@Test @Test
@ -173,13 +188,13 @@ public class ActorSystemTest {
}); });
} }
initLatch.countDown(); initLatch.countDown();
Assert.assertTrue(actorsReadyLatch.await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS)); Assertions.assertTrue(actorsReadyLatch.await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS));
actorSystem.tell(actorId, new IntTbActorMsg(42)); actorSystem.tell(actorId, new IntTbActorMsg(42));
Assert.assertTrue(testCtx.getLatch().await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS)); Assertions.assertTrue(testCtx.getLatch().await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS));
//One for creation and one for message //One for creation and one for message
Assert.assertEquals(2, testCtx.getInvocationCount().get()); Assertions.assertEquals(2, testCtx.getInvocationCount().get());
} }
@Test @Test
@ -197,9 +212,9 @@ public class ActorSystemTest {
actorId1.tell(new IntTbActorMsg(42)); actorId1.tell(new IntTbActorMsg(42));
actorId2.tell(new IntTbActorMsg(42)); actorId2.tell(new IntTbActorMsg(42));
Assert.assertFalse(testCtx1.getLatch().await(2, TimeUnit.SECONDS)); Assertions.assertFalse(testCtx1.getLatch().await(2, TimeUnit.SECONDS));
Assert.assertTrue(testCtx2.getLatch().await(1, TimeUnit.SECONDS)); Assertions.assertTrue(testCtx2.getLatch().await(1, TimeUnit.SECONDS));
Assert.assertTrue(testCtx1.getLatch().await(3, TimeUnit.SECONDS)); Assertions.assertTrue(testCtx1.getLatch().await(3, TimeUnit.SECONDS));
} }
@ -233,16 +248,16 @@ public class ActorSystemTest {
log.info("Submitted all messages"); log.info("Submitted all messages");
testCtxes.forEach(ctx -> { testCtxes.forEach(ctx -> {
try { try {
boolean success = ctx.getLatch().await(1, TimeUnit.MINUTES); boolean success = ctx.getLatch().await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS);
if (!success) { if (!success) {
log.warn("Failed: {}, {}", ctx.getActual().get(), ctx.getInvocationCount().get()); log.warn("Failed: {}, {}", ctx.getActual().get(), ctx.getInvocationCount().get());
} }
Assert.assertTrue(success); Assertions.assertTrue(success);
Assert.assertEquals(expected, ctx.getActual().get()); Assertions.assertEquals(expected, ctx.getActual().get());
Assert.assertEquals(msgNumber, ctx.getInvocationCount().get()); Assertions.assertEquals(msgNumber, ctx.getInvocationCount().get());
ctx.clear(); ctx.clear();
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); log.error("interrupted", e);
} }
}); });
long duration = System.nanoTime() - start; long duration = System.nanoTime() - start;

9
common/actor/src/test/java/org/thingsboard/server/actors/SlowCreateActor.java

@ -28,11 +28,13 @@ public class SlowCreateActor extends TestRootActor {
public SlowCreateActor(TbActorId actorId, ActorTestCtx testCtx, CountDownLatch initLatch) { public SlowCreateActor(TbActorId actorId, ActorTestCtx testCtx, CountDownLatch initLatch) {
super(actorId, testCtx); super(actorId, testCtx);
try { try {
log.info("awaiting on latch {} ...", initLatch);
initLatch.await(TIMEOUT_AWAIT_MAX_MS, TimeUnit.MILLISECONDS); initLatch.await(TIMEOUT_AWAIT_MAX_MS, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); log.error("interrupted", e);
} }
testCtx.getInvocationCount().incrementAndGet(); testCtx.getInvocationCount().incrementAndGet();
log.info("finished {} ...", initLatch);
} }
public static class SlowCreateActorCreator implements TbActorCreator { public static class SlowCreateActorCreator implements TbActorCreator {
@ -54,7 +56,10 @@ public class SlowCreateActor extends TestRootActor {
@Override @Override
public TbActor createActor() { public TbActor createActor() {
return new SlowCreateActor(actorId, testCtx, initLatch); log.info("creating slow actor...");
SlowCreateActor slowCreateActor = new SlowCreateActor(actorId, testCtx, initLatch);
log.info("created slow actor {}", slowCreateActor);
return slowCreateActor;
} }
} }
} }

3
common/actor/src/test/resources/junit-platform.properties

@ -0,0 +1,3 @@
junit.jupiter.execution.parallel.enabled = true
junit.jupiter.execution.parallel.mode.default = concurrent
junit.jupiter.execution.parallel.mode.classes.default = concurrent

0
common/actor/src/test/resources/logback.xml → common/actor/src/test/resources/logback-test.xml

5
common/cache/pom.xml

@ -97,11 +97,6 @@
<artifactId>spring-boot-starter-test</artifactId> <artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.awaitility</groupId> <groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId> <artifactId>awaitility</artifactId>

21
common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCacheEvictEvent.java

@ -0,0 +1,21 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.customer;
import org.thingsboard.server.common.data.id.TenantId;
public record CustomerCacheEvictEvent(TenantId tenantId, String newTitle, String oldTitle) {
}

42
common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCacheKey.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.customer;
import lombok.EqualsAndHashCode;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.thingsboard.server.common.data.id.TenantId;
import java.io.Serial;
import java.io.Serializable;
@EqualsAndHashCode
@RequiredArgsConstructor
public class CustomerCacheKey implements Serializable {
@Serial
private static final long serialVersionUID = 5706958428811356925L;
@NonNull
private final TenantId tenantId;
private final String title;
@Override
public String toString() {
return tenantId.getId() + "_" + title;
}
}

33
common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCaffeineCache.java

@ -0,0 +1,33 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.customer;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CaffeineTbTransactionalCache;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.Customer;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true)
@Service("CustomerCache")
public class CustomerCaffeineCache extends CaffeineTbTransactionalCache<CustomerCacheKey, Customer> {
public CustomerCaffeineCache(CacheManager cacheManager) {
super(cacheManager, CacheConstants.CUSTOMER_CACHE);
}
}

35
common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerRedisCache.java

@ -0,0 +1,35 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.customer;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CacheSpecsMap;
import org.thingsboard.server.cache.RedisTbTransactionalCache;
import org.thingsboard.server.cache.TBRedisCacheConfiguration;
import org.thingsboard.server.cache.TbJsonRedisSerializer;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.Customer;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis")
@Service("CustomerCache")
public class CustomerRedisCache extends RedisTbTransactionalCache<CustomerCacheKey, Customer> {
public CustomerRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) {
super(CacheConstants.CUSTOMER_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbJsonRedisSerializer<>(Customer.class));
}
}

11
common/cache/src/main/java/org/thingsboard/server/cache/limits/DefaultRateLimitService.java

@ -63,12 +63,21 @@ public class DefaultRateLimitService implements RateLimitService {
@Override @Override
public boolean checkRateLimit(LimitedApi api, TenantId tenantId, Object level) { public boolean checkRateLimit(LimitedApi api, TenantId tenantId, Object level) {
return checkRateLimit(api, tenantId, level, false);
}
@Override
public boolean checkRateLimit(LimitedApi api, TenantId tenantId, Object level, boolean ignoreTenantNotFound) {
if (tenantId.isSysTenantId()) { if (tenantId.isSysTenantId()) {
return true; return true;
} }
TenantProfile tenantProfile = tenantProfileProvider.get(tenantId); TenantProfile tenantProfile = tenantProfileProvider.get(tenantId);
if (tenantProfile == null) { if (tenantProfile == null) {
throw new TenantProfileNotFoundException(tenantId); if (ignoreTenantNotFound) {
return true;
} else {
throw new TenantProfileNotFoundException(tenantId);
}
} }
String rateLimitConfig = tenantProfile.getProfileConfiguration() String rateLimitConfig = tenantProfile.getProfileConfiguration()

2
common/cache/src/main/java/org/thingsboard/server/cache/limits/RateLimitService.java

@ -24,6 +24,8 @@ public interface RateLimitService {
boolean checkRateLimit(LimitedApi api, TenantId tenantId, Object level); boolean checkRateLimit(LimitedApi api, TenantId tenantId, Object level);
boolean checkRateLimit(LimitedApi api, TenantId tenantId, Object level, boolean ignoreTenantNotFound);
boolean checkRateLimit(LimitedApi api, Object level, String rateLimitConfig); boolean checkRateLimit(LimitedApi api, Object level, String rateLimitConfig);
void cleanUp(LimitedApi api, Object level); void cleanUp(LimitedApi api, Object level);

21
common/cache/src/main/java/org/thingsboard/server/cache/user/UserCacheEvictEvent.java

@ -0,0 +1,21 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.user;
import org.thingsboard.server.common.data.id.TenantId;
public record UserCacheEvictEvent(TenantId tenantId, String newEmail, String oldEmail) {
}

42
common/cache/src/main/java/org/thingsboard/server/cache/user/UserCacheKey.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.user;
import lombok.EqualsAndHashCode;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.thingsboard.server.common.data.id.TenantId;
import java.io.Serial;
import java.io.Serializable;
@EqualsAndHashCode
@RequiredArgsConstructor
public class UserCacheKey implements Serializable {
@Serial
private static final long serialVersionUID = 7357353074893750678L;
@NonNull
private final TenantId tenantId;
private final String email;
@Override
public String toString() {
return tenantId.getId() + "_" + email;
}
}

33
common/cache/src/main/java/org/thingsboard/server/cache/user/UserCaffeineCache.java

@ -0,0 +1,33 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.user;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CaffeineTbTransactionalCache;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.User;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true)
@Service("UserCache")
public class UserCaffeineCache extends CaffeineTbTransactionalCache<UserCacheKey, User> {
public UserCaffeineCache(CacheManager cacheManager) {
super(cacheManager, CacheConstants.USER_CACHE);
}
}

35
common/cache/src/main/java/org/thingsboard/server/cache/user/UserRedisCache.java

@ -0,0 +1,35 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.user;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CacheSpecsMap;
import org.thingsboard.server.cache.RedisTbTransactionalCache;
import org.thingsboard.server.cache.TBRedisCacheConfiguration;
import org.thingsboard.server.cache.TbJsonRedisSerializer;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.User;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis")
@Service("UserCache")
public class UserRedisCache extends RedisTbTransactionalCache<UserCacheKey, User> {
public UserRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) {
super(CacheConstants.USER_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbJsonRedisSerializer<>(User.class));
}
}

5
common/cache/src/test/java/org/thingsboard/server/cache/CacheSpecsMapTest.java

@ -26,6 +26,7 @@ import org.springframework.cache.support.SimpleCacheManager;
import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.thingsboard.server.common.data.CacheConstants;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
@ -53,8 +54,8 @@ public class CacheSpecsMapTest {
@Test @Test
public void givenCacheConfig_whenCacheManagerReady_thenVerifyExistedCachesWithNoTransactionAwareCacheDecorator() { public void givenCacheConfig_whenCacheManagerReady_thenVerifyExistedCachesWithNoTransactionAwareCacheDecorator() {
// We no longer use built-in transaction support for the caches, because we have our own cache cleanup and transaction logic that implements CAS. // We no longer use built-in transaction support for the caches, because we have our own cache cleanup and transaction logic that implements CAS.
assertThat(cacheManager.getCache("relations")).isInstanceOf(CaffeineCache.class); assertThat(cacheManager.getCache(CacheConstants.RELATIONS_CACHE)).isInstanceOf(CaffeineCache.class);
assertThat(cacheManager.getCache("devices")).isInstanceOf(CaffeineCache.class); assertThat(cacheManager.getCache(CacheConstants.DEVICE_CACHE)).isInstanceOf(CaffeineCache.class);
} }
@Test @Test

3
common/cache/src/test/resources/junit-platform.properties

@ -0,0 +1,3 @@
junit.jupiter.execution.parallel.enabled = true
junit.jupiter.execution.parallel.mode.default = concurrent
junit.jupiter.execution.parallel.mode.classes.default = concurrent

0
common/cache/src/test/resources/logback.xml → common/cache/src/test/resources/logback-test.xml

5
common/cluster-api/pom.xml

@ -109,11 +109,6 @@
<artifactId>spring-boot-starter-test</artifactId> <artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.awaitility</groupId> <groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId> <artifactId>awaitility</artifactId>

5
common/dao-api/pom.xml

@ -105,11 +105,6 @@
<artifactId>spring-boot-starter-test</artifactId> <artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.awaitility</groupId> <groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId> <artifactId>awaitility</artifactId>

2
common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java

@ -41,6 +41,8 @@ public interface AssetService extends EntityDaoService {
Asset findAssetByTenantIdAndName(TenantId tenantId, String name); Asset findAssetByTenantIdAndName(TenantId tenantId, String name);
ListenableFuture<Asset> findAssetByTenantIdAndNameAsync(TenantId tenantId, String name);
Asset saveAsset(Asset asset, boolean doValidate); Asset saveAsset(Asset asset, boolean doValidate);
Asset saveAsset(Asset asset); Asset saveAsset(Asset asset);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/customer/CustomerService.java

@ -31,6 +31,8 @@ public interface CustomerService extends EntityDaoService {
Optional<Customer> findCustomerByTenantIdAndTitle(TenantId tenantId, String title); Optional<Customer> findCustomerByTenantIdAndTitle(TenantId tenantId, String title);
ListenableFuture<Optional<Customer>> findCustomerByTenantIdAndTitleAsync(TenantId tenantId, String title);
ListenableFuture<Customer> findCustomerByIdAsync(TenantId tenantId, CustomerId customerId); ListenableFuture<Customer> findCustomerByIdAsync(TenantId tenantId, CustomerId customerId);
Customer saveCustomer(Customer customer); Customer saveCustomer(Customer customer);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java

@ -72,6 +72,8 @@ public interface DashboardService extends EntityDaoService {
DashboardInfo findFirstDashboardInfoByTenantIdAndName(TenantId tenantId, String name); DashboardInfo findFirstDashboardInfoByTenantIdAndName(TenantId tenantId, String name);
ListenableFuture<DashboardInfo> findFirstDashboardInfoByTenantIdAndNameAsync(TenantId tenantId, String name);
List<Dashboard> findTenantDashboardsByTitle(TenantId tenantId, String title); List<Dashboard> findTenantDashboardsByTitle(TenantId tenantId, String title);
} }

2
common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java

@ -49,6 +49,8 @@ public interface DeviceService extends EntityDaoService {
Device findDeviceByTenantIdAndName(TenantId tenantId, String name); Device findDeviceByTenantIdAndName(TenantId tenantId, String name);
ListenableFuture<Device> findDeviceByTenantIdAndNameAsync(TenantId tenantId, String name);
Device saveDevice(Device device); Device saveDevice(Device device);
Device saveDevice(Device device, boolean doValidate); Device saveDevice(Device device, boolean doValidate);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java

@ -42,6 +42,8 @@ public interface EdgeService extends EntityDaoService {
Edge findEdgeByTenantIdAndName(TenantId tenantId, String name); Edge findEdgeByTenantIdAndName(TenantId tenantId, String name);
ListenableFuture<Edge> findEdgeByTenantIdAndNameAsync(TenantId tenantId, String name);
Optional<Edge> findEdgeByRoutingKey(TenantId tenantId, String routingKey); Optional<Edge> findEdgeByRoutingKey(TenantId tenantId, String routingKey);
Edge saveEdge(Edge edge); Edge saveEdge(Edge edge);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java

@ -54,6 +54,8 @@ public interface EntityViewService extends EntityDaoService {
EntityView findEntityViewByTenantIdAndName(TenantId tenantId, String name); EntityView findEntityViewByTenantIdAndName(TenantId tenantId, String name);
ListenableFuture<EntityView> findEntityViewByTenantIdAndNameAsync(TenantId tenantId, String name);
PageData<EntityView> findEntityViewByTenantId(TenantId tenantId, PageLink pageLink); PageData<EntityView> findEntityViewByTenantId(TenantId tenantId, PageLink pageLink);
PageData<EntityViewInfo> findEntityViewInfosByTenantId(TenantId tenantId, PageLink pageLink); PageData<EntityViewInfo> findEntityViewInfosByTenantId(TenantId tenantId, PageLink pageLink);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java

@ -41,6 +41,8 @@ public interface UserService extends EntityDaoService {
User findUserByTenantIdAndEmail(TenantId tenantId, String email); User findUserByTenantIdAndEmail(TenantId tenantId, String email);
ListenableFuture<User> findUserByTenantIdAndEmailAsync(TenantId tenantId, String email);
User saveUser(TenantId tenantId, User user); User saveUser(TenantId tenantId, User user);
UserCredentials findUserCredentialsByUserId(TenantId tenantId, UserId userId); UserCredentials findUserCredentialsByUserId(TenantId tenantId, UserId userId);

5
common/data/pom.xml

@ -73,11 +73,6 @@
<artifactId>spring-boot-starter-test</artifactId> <artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.awaitility</groupId> <groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId> <artifactId>awaitility</artifactId>

2
common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java

@ -21,6 +21,8 @@ public class CacheConstants {
public static final String DEVICE_CACHE = "devices"; public static final String DEVICE_CACHE = "devices";
public static final String SESSIONS_CACHE = "sessions"; public static final String SESSIONS_CACHE = "sessions";
public static final String ASSET_CACHE = "assets"; public static final String ASSET_CACHE = "assets";
public static final String CUSTOMER_CACHE = "customers";
public static final String USER_CACHE = "users";
public static final String ENTITY_VIEW_CACHE = "entityViews"; public static final String ENTITY_VIEW_CACHE = "entityViews";
public static final String EDGE_CACHE = "edges"; public static final String EDGE_CACHE = "edges";
public static final String CLAIM_DEVICES_CACHE = "claimDevices"; public static final String CLAIM_DEVICES_CACHE = "claimDevices";

4
common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java

@ -18,6 +18,7 @@ package org.thingsboard.server.common.data;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.Getter; import lombok.Getter;
@ -33,7 +34,6 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.Length;
import org.thingsboard.server.common.data.validation.NoXss; import org.thingsboard.server.common.data.validation.NoXss;
import jakarta.validation.Valid;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
@ -42,7 +42,7 @@ import java.io.IOException;
@ToString(exclude = {"image", "profileDataBytes"}) @ToString(exclude = {"image", "profileDataBytes"})
@EqualsAndHashCode(callSuper = true) @EqualsAndHashCode(callSuper = true)
@Slf4j @Slf4j
public class DeviceProfile extends BaseData<DeviceProfileId> implements HasName, HasTenantId, HasOtaPackage, HasRuleEngineProfile, ExportableEntity<DeviceProfileId>, HasImage { public class DeviceProfile extends BaseData<DeviceProfileId> implements HasName, HasTenantId, HasOtaPackage, HasRuleEngineProfile, ExportableEntity<DeviceProfileId>, HasImage, HasDefaultOption {
private static final long serialVersionUID = 6998485460273302018L; private static final long serialVersionUID = 6998485460273302018L;

13
netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/IntegrationTestSuite.java → common/data/src/main/java/org/thingsboard/server/common/data/HasDefaultOption.java

@ -13,15 +13,10 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.mqtt.integration; package org.thingsboard.server.common.data;
import org.junit.extensions.cpsuite.ClasspathSuite; public interface HasDefaultOption {
import org.junit.runner.RunWith;
@RunWith(ClasspathSuite.class) boolean isDefault();
@ClasspathSuite.ClassnameFilters({
"org.thingsboard.mqtt.integration.*Test",
})
public class IntegrationTestSuite {
} }

3
common/data/src/main/java/org/thingsboard/server/common/data/asset/AssetProfile.java

@ -22,6 +22,7 @@ import lombok.ToString;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.BaseData;
import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasDefaultOption;
import org.thingsboard.server.common.data.HasImage; import org.thingsboard.server.common.data.HasImage;
import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HasRuleEngineProfile; import org.thingsboard.server.common.data.HasRuleEngineProfile;
@ -38,7 +39,7 @@ import org.thingsboard.server.common.data.validation.NoXss;
@ToString(exclude = {"image"}) @ToString(exclude = {"image"})
@EqualsAndHashCode(callSuper = true) @EqualsAndHashCode(callSuper = true)
@Slf4j @Slf4j
public class AssetProfile extends BaseData<AssetProfileId> implements HasName, HasTenantId, HasRuleEngineProfile, ExportableEntity<AssetProfileId>, HasImage { public class AssetProfile extends BaseData<AssetProfileId> implements HasName, HasTenantId, HasRuleEngineProfile, ExportableEntity<AssetProfileId>, HasImage, HasDefaultOption {
private static final long serialVersionUID = 6998485460273302018L; private static final long serialVersionUID = 6998485460273302018L;

1
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/lwm2m/OtherConfiguration.java

@ -39,4 +39,5 @@ public class OtherConfiguration extends PowerSavingConfiguration {
private Long pagingTransmissionWindow; private Long pagingTransmissionWindow;
private String fwUpdateResource; private String fwUpdateResource;
private String swUpdateResource; private String swUpdateResource;
private String defaultObjectIDVer;
} }

30
common/data/src/main/java/org/thingsboard/server/common/data/exception/RateLimitExceededException.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.exception;
import org.thingsboard.server.common.data.limit.LimitedApi;
public class RateLimitExceededException extends AbstractRateLimitException {
public RateLimitExceededException(String message) {
super(message);
}
public RateLimitExceededException(LimitedApi api) {
super("Rate limit for " + api.getLabel() + " is exceeded");
}
}

3
common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java

@ -39,7 +39,8 @@ public enum LimitedApi {
TWO_FA_VERIFICATION_CODE_SEND(false, true), TWO_FA_VERIFICATION_CODE_SEND(false, true),
TWO_FA_VERIFICATION_CODE_CHECK(false, true), TWO_FA_VERIFICATION_CODE_CHECK(false, true),
TRANSPORT_MESSAGES_PER_TENANT("transport messages", true), TRANSPORT_MESSAGES_PER_TENANT("transport messages", true),
TRANSPORT_MESSAGES_PER_DEVICE("transport messages per device", false); TRANSPORT_MESSAGES_PER_DEVICE("transport messages per device", false),
EMAILS("emails sending", true);
private Function<DefaultTenantProfileConfiguration, String> configExtractor; private Function<DefaultTenantProfileConfiguration, String> configExtractor;
@Getter @Getter

2
common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java

@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.validation.NoXss;
import java.util.Optional; import java.util.Optional;
@Data @Data
public class Queue extends BaseDataWithAdditionalInfo<QueueId> implements HasName, HasTenantId { public class Queue extends BaseDataWithAdditionalInfo<QueueId> implements HasName, HasTenantId, QueueConfig {
private TenantId tenantId; private TenantId tenantId;
@NoXss @NoXss
@Length(fieldName = "name") @Length(fieldName = "name")

12
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntityContainer.java → common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java

@ -13,16 +13,12 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.rule.engine.util; package org.thingsboard.server.common.data.queue;
import lombok.Data; public interface QueueConfig {
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.EntityId;
@Data boolean isConsumerPerPartition();
public class EntityContainer {
private EntityId entityId; int getPollInterval();
private EntityType entityType;
} }

9
common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java

@ -23,6 +23,7 @@ import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo;
import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasDefaultOption;
import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
@ -35,7 +36,7 @@ import org.thingsboard.server.common.data.validation.NoXss;
@Data @Data
@EqualsAndHashCode(callSuper = true) @EqualsAndHashCode(callSuper = true)
@Slf4j @Slf4j
public class RuleChain extends BaseDataWithAdditionalInfo<RuleChainId> implements HasName, HasTenantId, ExportableEntity<RuleChainId> { public class RuleChain extends BaseDataWithAdditionalInfo<RuleChainId> implements HasName, HasTenantId, ExportableEntity<RuleChainId>, HasDefaultOption {
private static final long serialVersionUID = -5656679015121935465L; private static final long serialVersionUID = -5656679015121935465L;
@ -108,4 +109,10 @@ public class RuleChain extends BaseDataWithAdditionalInfo<RuleChainId> implement
setJson(data, json -> this.configuration = json, bytes -> this.configurationBytes = bytes); setJson(data, json -> this.configuration = json, bytes -> this.configurationBytes = bytes);
} }
@JsonIgnore
@Override
public boolean isDefault() {
return root && type == RuleChainType.CORE;
}
} }

2
common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtSettings.java

@ -48,7 +48,7 @@ public class JwtSettings {
* Key is used to sign {@link JwtToken}. * Key is used to sign {@link JwtToken}.
* Base64 encoded * Base64 encoded
*/ */
@Schema(description = "The JWT key is used to sing token. Base64 encoded.", example = "cTU4WnNqemI2aU5wbWVjdm1vYXRzanhjNHRUcXliMjE=") @Schema(description = "The JWT key is used to sing token. Base64 encoded.", example = "dkVTUzU2M2VMWUNwVVltTUhQU2o5SUM0Tkc3M0k2Ykdwcm85QTl6R0RaQ252OFlmVDk2OEptZXBNcndGeExFZg==")
private String tokenSigningKey; private String tokenSigningKey;
} }

14
common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java

@ -19,18 +19,18 @@ import com.github.os72.protobuf.dynamic.DynamicSchema;
import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage; import com.google.protobuf.DynamicMessage;
import com.squareup.wire.schema.internal.parser.ProtoFileElement; import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import org.junit.Test; import org.junit.jupiter.api.Test;
import org.junit.runner.RunWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.MockitoJUnitRunner; import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
@RunWith(MockitoJUnitRunner.class) @ExtendWith(MockitoExtension.class)
public class DynamicProtoUtilsTest { public class DynamicProtoUtilsTest {
@Test @Test

17
common/data/src/test/java/org/thingsboard/server/common/data/UUIDConverterTest.java

@ -16,11 +16,10 @@
package org.thingsboard.server.common.data; package org.thingsboard.server.common.data;
import com.datastax.oss.driver.api.core.uuid.Uuids; import com.datastax.oss.driver.api.core.uuid.Uuids;
import org.junit.Assert; import org.junit.jupiter.api.Test;
import org.junit.Test;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.runner.RunWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.MockitoJUnitRunner; import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Random; import java.util.Random;
import java.util.UUID; import java.util.UUID;
@ -28,14 +27,14 @@ import java.util.UUID;
/** /**
* Created by ashvayka on 14.07.17. * Created by ashvayka on 14.07.17.
*/ */
@RunWith(MockitoJUnitRunner.class) @ExtendWith(MockitoExtension .class)
public class UUIDConverterTest { public class UUIDConverterTest {
@Test @Test
public void basicUuidToStringTest() { public void basicUuidToStringTest() {
UUID original = UUID.fromString("58e0a7d7-eebc-11d8-9669-0800200c9a66"); UUID original = UUID.fromString("58e0a7d7-eebc-11d8-9669-0800200c9a66");
String result = UUIDConverter.fromTimeUUID(original); String result = UUIDConverter.fromTimeUUID(original);
Assert.assertEquals("1d8eebc58e0a7d796690800200c9a66", result); Assertions.assertEquals("1d8eebc58e0a7d796690800200c9a66", result);
} }
@ -47,13 +46,13 @@ public class UUIDConverterTest {
@Test @Test
public void basicUuidConversion() { public void basicUuidConversion() {
UUID original = UUID.fromString("3dd11790-abf2-11ea-b151-83a091b9d4cc"); UUID original = UUID.fromString("3dd11790-abf2-11ea-b151-83a091b9d4cc");
Assert.assertEquals(Uuids.unixTimestamp(original), 1591886749577L); Assertions.assertEquals(Uuids.unixTimestamp(original), 1591886749577L);
} }
@Test @Test
public void basicStringToUUIDTest() { public void basicStringToUUIDTest() {
UUID result = UUIDConverter.fromString("1d8eebc58e0a7d796690800200c9a66"); UUID result = UUIDConverter.fromString("1d8eebc58e0a7d796690800200c9a66");
Assert.assertEquals(UUID.fromString("58e0a7d7-eebc-11d8-9669-0800200c9a66"), result); Assertions.assertEquals(UUID.fromString("58e0a7d7-eebc-11d8-9669-0800200c9a66"), result);
} }
@Test @Test
@ -83,7 +82,7 @@ public class UUIDConverterTest {
System.out.println("Before: " + before + " | " + beforeStr); System.out.println("Before: " + before + " | " + beforeStr);
System.out.println("After: " + after + " | " + afterStr); System.out.println("After: " + after + " | " + afterStr);
} }
Assert.assertTrue(afterStr.compareTo(beforeStr) >= 0); Assertions.assertTrue(afterStr.compareTo(beforeStr) >= 0);
} }
} }

6
common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java

@ -15,14 +15,14 @@
*/ */
package org.thingsboard.server.common.data.id; package org.thingsboard.server.common.data.id;
import org.junit.Assert; import org.junit.jupiter.api.Assertions;
import org.junit.Test; import org.junit.jupiter.api.Test;
public class EntityIdTest { public class EntityIdTest {
@Test @Test
public void givenConstantNullUuid_whenCompare_thenToStringEqualsPredefinedUuid() { public void givenConstantNullUuid_whenCompare_thenToStringEqualsPredefinedUuid() {
Assert.assertEquals("13814000-1dd2-11b2-8080-808080808080", EntityId.NULL_UUID.toString()); Assertions.assertEquals("13814000-1dd2-11b2-8080-808080808080", EntityId.NULL_UUID.toString());
} }
} }

3
common/data/src/test/resources/junit-platform.properties

@ -0,0 +1,3 @@
junit.jupiter.execution.parallel.enabled = true
junit.jupiter.execution.parallel.mode.default = concurrent
junit.jupiter.execution.parallel.mode.classes.default = concurrent

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save