Browse Source

Merge branch 'master' into master

pull/10748/head
cogic 2 years ago
committed by GitHub
parent
commit
c3bab5bb02
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 5
      application/src/main/data/json/system/widget_bundles/charts.json
  2. 2
      application/src/main/data/json/system/widget_types/alarms_table.json
  3. 2
      application/src/main/data/json/system/widget_types/asset_admin_table.json
  4. 2
      application/src/main/data/json/system/widget_types/device_admin_table.json
  5. 2
      application/src/main/data/json/system/widget_types/entities_hierarchy.json
  6. 2
      application/src/main/data/json/system/widget_types/entities_table.json
  7. 33
      application/src/main/data/json/system/widget_types/radar.json
  8. 25
      application/src/main/data/json/system/widget_types/radar_deprecated.json
  9. 2
      application/src/main/data/json/system/widget_types/timeseries_table.json
  10. 17
      application/src/main/data/json/tenant/dashboards/gateways.json
  11. 25
      application/src/main/data/upgrade/3.6.4/schema_update.sql
  12. 4
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java
  13. 28
      application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java
  14. 3
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  15. 5
      application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java
  16. 2
      application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java
  17. 2
      application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java
  18. 135
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  19. 2
      application/src/main/java/org/thingsboard/server/service/install/EntityDatabaseSchemaService.java
  20. 10
      application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java
  21. 7
      application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java
  22. 2
      application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java
  23. 1
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java
  24. 82
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
  25. 43
      application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java
  26. 37
      application/src/main/java/org/thingsboard/server/service/mail/MailSenderInternalExecutorService.java
  27. 1
      application/src/main/java/org/thingsboard/server/service/mail/TbMailContextComponent.java
  28. 3
      application/src/main/java/org/thingsboard/server/service/notification/channels/MobileAppNotificationChannel.java
  29. 416
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  30. 41
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  31. 322
      application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java
  32. 131
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  33. 8
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java
  34. 7
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java
  35. 25
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java
  36. 290
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java
  37. 80
      application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsService.java
  38. 7
      application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java
  39. 4
      application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsService.java
  40. 72
      application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java
  41. 6
      application/src/main/java/org/thingsboard/server/service/security/model/token/OAuth2AppTokenFactory.java
  42. 28
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java
  43. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java
  44. 21
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
  45. 10
      application/src/main/resources/thingsboard.yml
  46. 17
      application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
  47. 4
      application/src/test/java/org/thingsboard/server/controller/AdminControllerTest.java
  48. 7
      application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java
  49. 6
      application/src/test/java/org/thingsboard/server/controller/AuthControllerTest.java
  50. 27
      application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java
  51. 38
      application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java
  52. 13
      application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java
  53. 2
      application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java
  54. 62
      application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java
  55. 35
      application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java
  56. 17
      application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java
  57. 5
      common/actor/pom.xml
  58. 69
      common/actor/src/test/java/org/thingsboard/server/actors/ActorSystemTest.java
  59. 9
      common/actor/src/test/java/org/thingsboard/server/actors/SlowCreateActor.java
  60. 3
      common/actor/src/test/resources/junit-platform.properties
  61. 0
      common/actor/src/test/resources/logback-test.xml
  62. 5
      common/cache/pom.xml
  63. 21
      common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCacheEvictEvent.java
  64. 42
      common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCacheKey.java
  65. 33
      common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCaffeineCache.java
  66. 35
      common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerRedisCache.java
  67. 11
      common/cache/src/main/java/org/thingsboard/server/cache/limits/DefaultRateLimitService.java
  68. 2
      common/cache/src/main/java/org/thingsboard/server/cache/limits/RateLimitService.java
  69. 21
      common/cache/src/main/java/org/thingsboard/server/cache/user/UserCacheEvictEvent.java
  70. 42
      common/cache/src/main/java/org/thingsboard/server/cache/user/UserCacheKey.java
  71. 33
      common/cache/src/main/java/org/thingsboard/server/cache/user/UserCaffeineCache.java
  72. 35
      common/cache/src/main/java/org/thingsboard/server/cache/user/UserRedisCache.java
  73. 5
      common/cache/src/test/java/org/thingsboard/server/cache/CacheSpecsMapTest.java
  74. 3
      common/cache/src/test/resources/junit-platform.properties
  75. 0
      common/cache/src/test/resources/logback-test.xml
  76. 5
      common/cluster-api/pom.xml
  77. 5
      common/dao-api/pom.xml
  78. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java
  79. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/customer/CustomerService.java
  80. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java
  81. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java
  82. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java
  83. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java
  84. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java
  85. 5
      common/data/pom.xml
  86. 2
      common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java
  87. 4
      common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java
  88. 13
      common/data/src/main/java/org/thingsboard/server/common/data/HasDefaultOption.java
  89. 3
      common/data/src/main/java/org/thingsboard/server/common/data/asset/AssetProfile.java
  90. 1
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/lwm2m/OtherConfiguration.java
  91. 30
      common/data/src/main/java/org/thingsboard/server/common/data/exception/RateLimitExceededException.java
  92. 3
      common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java
  93. 2
      common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java
  94. 12
      common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java
  95. 9
      common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java
  96. 2
      common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtSettings.java
  97. 14
      common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java
  98. 17
      common/data/src/test/java/org/thingsboard/server/common/data/UUIDConverterTest.java
  99. 6
      common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java
  100. 3
      common/data/src/test/resources/junit-platform.properties

5
application/src/main/data/json/system/widget_bundles/charts.json

@ -21,7 +21,7 @@
"doughnut",
"horizontal_doughnut",
"polar_area",
"charts.radar_chart_js",
"radar",
"charts.basic_timeseries",
"charts.state_chart",
"charts.timeseries_bars_flot",
@ -29,6 +29,7 @@
"charts.pie",
"charts.pie_chart_js",
"charts.doughnut_chart_js",
"charts.polar_area_chart_js"
"charts.polar_area_chart_js",
"charts.radar_chart_js"
]
}

2
application/src/main/data/json/system/widget_types/alarms_table.json

@ -11,7 +11,7 @@
"resources": [],
"templateHtml": "<tb-alarms-table-widget \n [ctx]=\"ctx\">\n</tb-alarms-table-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.alarmsTableWidget.onDataUpdated();\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.alarmsTableWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.alarmsTableWidget.onEditModeChanged();\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'cellClick': {\n name: 'widget-action.cell-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"settingsDirective": "tb-alarms-table-widget-settings",

2
application/src/main/data/json/system/widget_types/asset_admin_table.json

@ -11,7 +11,7 @@
"resources": [],
"templateHtml": "<tb-entities-table-widget \n [ctx]=\"ctx\">\n</tb-entities-table-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.entitiesTableWidget.onEditModeChanged();\n}\n\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n },\n 'cellClick': {\n name: 'widget-action.cell-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"settingsDirective": "tb-entities-table-widget-settings",

2
application/src/main/data/json/system/widget_types/device_admin_table.json

@ -11,7 +11,7 @@
"resources": [],
"templateHtml": "<tb-entities-table-widget \n [ctx]=\"ctx\">\n</tb-entities-table-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.entitiesTableWidget.onEditModeChanged();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n },\n 'cellClick': {\n name: 'widget-action.cell-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"settingsDirective": "tb-entities-table-widget-settings",

2
application/src/main/data/json/system/widget_types/entities_hierarchy.json

@ -11,7 +11,7 @@
"resources": [],
"templateHtml": "<tb-entities-hierarchy-widget \n [ctx]=\"ctx\">\n</tb-entities-hierarchy-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesHierarchyWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'nodeSelected': {\n name: 'widget-action.node-selected',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesHierarchyWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.entitiesHierarchyWidget.onEditModeChanged();\n}\n\nself.typeParameters = function() {\n return {\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'nodeSelected': {\n name: 'widget-action.node-selected',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"settingsDirective": "tb-entities-hierarchy-widget-settings",

2
application/src/main/data/json/system/widget_types/entities_table.json

@ -11,7 +11,7 @@
"resources": [],
"templateHtml": "<tb-entities-table-widget \n [ctx]=\"ctx\">\n</tb-entities-table-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'name', type: 'entityField' }];\n }\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.entitiesTableWidget.onEditModeChanged();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'name', type: 'entityField' }];\n }\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n },\n 'cellClick': {\n name: 'widget-action.cell-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"settingsDirective": "tb-entities-table-widget-settings",

33
application/src/main/data/json/system/widget_types/radar.json

File diff suppressed because one or more lines are too long

25
application/src/main/data/json/system/widget_types/radar_deprecated.json

File diff suppressed because one or more lines are too long

2
application/src/main/data/json/system/widget_types/timeseries_table.json

@ -11,7 +11,7 @@
"resources": [],
"templateHtml": "<tb-timeseries-table-widget \n [ctx]=\"ctx\">\n</tb-timeseries-table-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeseriesTableWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeseriesTableWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n ignoreDataUpdateOnIntervalTick: true,\n hasAdditionalLatestDataKeys: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeseriesTableWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeseriesTableWidget.onLatestDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.timeseriesTableWidget.onEditModeChanged();\n}\n\nself.typeParameters = function() {\n return {\n ignoreDataUpdateOnIntervalTick: true,\n hasAdditionalLatestDataKeys: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"latestDataKeySettingsSchema": "",

17
application/src/main/data/json/tenant/dashboards/gateways.json

File diff suppressed because one or more lines are too long

25
application/src/main/data/upgrade/3.6.4/schema_update.sql

@ -14,6 +14,23 @@
-- limitations under the License.
--
-- UPDATE PUBLIC CUSTOMERS START
DO
$$
BEGIN
IF NOT EXISTS (
SELECT FROM information_schema.columns
WHERE table_name = 'customer' AND column_name = 'is_public'
) THEN
ALTER TABLE customer ADD COLUMN is_public boolean DEFAULT false;
UPDATE customer SET is_public = true WHERE title = 'Public';
END IF;
END;
$$;
-- UPDATE PUBLIC CUSTOMERS END
-- create new attribute_kv table schema
DO
$$
@ -90,13 +107,7 @@ BEGIN
SELECT COUNT(*) INTO row_num_old FROM attribute_kv_old;
SELECT COUNT(*) INTO row_num FROM attribute_kv;
RAISE NOTICE 'Migrated % of % rows', row_num, row_num_old;
IF row_num != 0 THEN
DROP TABLE IF EXISTS attribute_kv_old;
ELSE
RAISE EXCEPTION 'Table attribute_kv is empty';
END IF;
DROP TABLE IF EXISTS attribute_kv_old;
CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);
END IF;
EXCEPTION

4
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java

@ -103,7 +103,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
RuleChain ruleChain = service.findRuleChainById(tenantId, entityId);
if (ruleChain != null && RuleChainType.CORE.equals(ruleChain.getType())) {
List<RuleNode> ruleNodeList = service.getRuleChainNodes(tenantId, entityId);
log.trace("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size());
log.debug("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size());
// Creating and starting the actors;
for (RuleNode ruleNode : ruleNodeList) {
log.trace("[{}][{}] Creating rule node [{}]: {}", entityId, ruleNode.getId(), ruleNode.getName(), ruleNode);
@ -124,7 +124,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
if (ruleChain != null && RuleChainType.CORE.equals(ruleChain.getType())) {
ruleChainName = ruleChain.getName();
List<RuleNode> ruleNodeList = service.getRuleChainNodes(tenantId, entityId);
log.trace("[{}][{}] Updating rule chain with {} nodes", tenantId, entityId, ruleNodeList.size());
log.debug("[{}][{}] Updating rule chain with {} nodes", tenantId, entityId, ruleNodeList.size());
for (RuleNode ruleNode : ruleNodeList) {
RuleNodeCtx existing = nodeActors.get(ruleNode.getId());
if (existing == null) {

28
application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java

@ -72,12 +72,11 @@ public class ThingsboardSecurityConfiguration {
public static final String JWT_TOKEN_HEADER_PARAM_V2 = "Authorization";
public static final String JWT_TOKEN_QUERY_PARAM = "token";
public static final String WEBJARS_ENTRY_POINT = "/webjars/**";
public static final String DEVICE_API_ENTRY_POINT = "/api/v1/**";
public static final String FORM_BASED_LOGIN_ENTRY_POINT = "/api/auth/login";
public static final String PUBLIC_LOGIN_ENTRY_POINT = "/api/auth/login/public";
public static final String TOKEN_REFRESH_ENTRY_POINT = "/api/auth/token";
protected static final String[] NON_TOKEN_BASED_AUTH_ENTRY_POINTS = new String[] {"/index.html", "/assets/**", "/static/**", "/api/noauth/**", "/webjars/**", "/api/license/**", "/api/images/public/**"};
protected static final String[] NON_TOKEN_BASED_AUTH_ENTRY_POINTS = new String[]{"/index.html", "/assets/**", "/static/**", "/api/noauth/**", "/webjars/**", "/api/license/**", "/api/images/public/**"};
public static final String TOKEN_BASED_AUTH_ENTRY_POINT = "/api/**";
public static final String WS_ENTRY_POINT = "/api/ws/**";
public static final String MAIL_OAUTH2_PROCESSING_ENTRY_POINT = "/api/admin/mail/oauth2/code";
@ -153,7 +152,7 @@ public class ThingsboardSecurityConfiguration {
protected JwtTokenAuthenticationProcessingFilter buildJwtTokenAuthenticationProcessingFilter() throws Exception {
List<String> pathsToSkip = new ArrayList<>(Arrays.asList(NON_TOKEN_BASED_AUTH_ENTRY_POINTS));
pathsToSkip.addAll(Arrays.asList(WS_ENTRY_POINT, TOKEN_REFRESH_ENTRY_POINT, FORM_BASED_LOGIN_ENTRY_POINT,
PUBLIC_LOGIN_ENTRY_POINT, DEVICE_API_ENTRY_POINT, WEBJARS_ENTRY_POINT, MAIL_OAUTH2_PROCESSING_ENTRY_POINT,
PUBLIC_LOGIN_ENTRY_POINT, DEVICE_API_ENTRY_POINT, MAIL_OAUTH2_PROCESSING_ENTRY_POINT,
DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT));
SkipPathRequestMatcher matcher = new SkipPathRequestMatcher(pathsToSkip, TOKEN_BASED_AUTH_ENTRY_POINT);
JwtTokenAuthenticationProcessingFilter filter
@ -210,18 +209,17 @@ public class ThingsboardSecurityConfiguration {
.exceptionHandling(config -> {})
.sessionManagement(config -> config.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
.authorizeHttpRequests(config -> config
.requestMatchers(WEBJARS_ENTRY_POINT).permitAll() // Webjars
.requestMatchers(DEVICE_API_ENTRY_POINT).permitAll() // Device HTTP Transport API
.requestMatchers(FORM_BASED_LOGIN_ENTRY_POINT).permitAll() // Login end-point
.requestMatchers(PUBLIC_LOGIN_ENTRY_POINT).permitAll() // Public login end-point
.requestMatchers(TOKEN_REFRESH_ENTRY_POINT).permitAll() // Token refresh end-point
.requestMatchers(MAIL_OAUTH2_PROCESSING_ENTRY_POINT).permitAll() // Mail oauth2 code processing url
.requestMatchers(DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT).permitAll() // Device connectivity certificate (public)
.requestMatchers(NON_TOKEN_BASED_AUTH_ENTRY_POINTS).permitAll() // static resources, user activation and password reset end-points
.requestMatchers(WS_ENTRY_POINT).permitAll() // Protected WebSocket API End-points
.requestMatchers(TOKEN_BASED_AUTH_ENTRY_POINT).authenticated()) // Protected API End-points
.formLogin(form -> form
.loginPage("/login").permitAll())
.requestMatchers(NON_TOKEN_BASED_AUTH_ENTRY_POINTS).permitAll() // static resources, user activation and password reset end-points (webjars included)
.requestMatchers(
DEVICE_API_ENTRY_POINT, // Device HTTP Transport API
FORM_BASED_LOGIN_ENTRY_POINT, // Login end-point
PUBLIC_LOGIN_ENTRY_POINT, // Public login end-point
TOKEN_REFRESH_ENTRY_POINT, // Token refresh end-point
MAIL_OAUTH2_PROCESSING_ENTRY_POINT, // Mail oauth2 code processing url
DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT, // Device connectivity certificate (public)
WS_ENTRY_POINT).permitAll() // Protected WebSocket API End-points
.requestMatchers(TOKEN_BASED_AUTH_ENTRY_POINT).authenticated() // Protected API End-points
.anyRequest().permitAll())
.exceptionHandling(config -> config.accessDeniedHandler(restAccessDeniedHandler))
.addFilterBefore(buildRestLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(buildRestPublicLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class)

3
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -133,7 +133,10 @@ public class ThingsboardInstallService {
case "3.6.4":
log.info("Upgrading ThingsBoard from version 3.6.4 to 3.7.0 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.6.4");
dataUpdateService.updateData("3.6.4");
entityDatabaseSchemaService.createCustomerTitleUniqueConstraintIfNotExists();
systemDataLoaderService.updateDefaultNotificationConfigs(false);
systemDataLoaderService.updateJwtSettings();
//TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache
break;
default:

5
application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java

@ -33,7 +33,6 @@ import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MClientCredential;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode;
import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration;
import org.thingsboard.server.common.data.device.data.DeviceData;
import org.thingsboard.server.common.data.device.data.PowerMode;
import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration;
@ -68,6 +67,8 @@ import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import static org.eclipse.leshan.core.LwM2m.Version.V1_0;
@Service
@TbCoreComponent
@RequiredArgsConstructor
@ -256,7 +257,7 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
Lwm2mDeviceProfileTransportConfiguration transportConfiguration = new Lwm2mDeviceProfileTransportConfiguration();
transportConfiguration.setBootstrap(Collections.emptyList());
transportConfiguration.setClientLwM2mSettings(new OtherConfiguration(1, 1, 1, PowerMode.DRX, null, null, null, null, null));
transportConfiguration.setClientLwM2mSettings(new OtherConfiguration(1, 1, 1, PowerMode.DRX, null, null, null, null, null, V1_0.toString()));
transportConfiguration.setObserveAttr(new TelemetryMappingConfiguration(Collections.emptyMap(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptyMap()));
DeviceProfileData deviceProfileData = new DeviceProfileData();

2
application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java

@ -27,11 +27,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperService
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.housekeeper.HousekeeperConfig;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.queue.consumer.QueueConsumerManager;
import javax.annotation.PreDestroy;
import java.util.LinkedHashSet;

2
application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java

@ -27,13 +27,13 @@ import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.housekeeper.HousekeeperConfig;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor;
import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService;
import org.thingsboard.server.service.queue.consumer.QueueConsumerManager;
import javax.annotation.PreDestroy;
import java.util.List;

135
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -19,13 +19,17 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import jakarta.annotation.Nullable;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Profile;
import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
import org.springframework.stereotype.Service;
@ -81,6 +85,7 @@ import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration;
@ -100,14 +105,11 @@ import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService;
import jakarta.annotation.Nullable;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.List;
import java.util.TreeMap;
@ -117,79 +119,51 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.thingsboard.server.common.data.DataConstants.DEFAULT_DEVICE_TYPE;
import static org.thingsboard.server.service.security.auth.jwt.settings.DefaultJwtSettingsService.isSigningKeyDefault;
import static org.thingsboard.server.service.security.auth.jwt.settings.DefaultJwtSettingsService.validateTokenSigningKeyLength;
@Service
@Profile("install")
@Slf4j
@RequiredArgsConstructor
public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
public static final String CUSTOMER_CRED = "customer";
public static final String ACTIVITY_STATE = "active";
@Autowired
private InstallScripts installScripts;
private final InstallScripts installScripts;
private final UserService userService;
private final AdminSettingsService adminSettingsService;
private final TenantService tenantService;
private final TenantProfileService tenantProfileService;
private final CustomerService customerService;
private final DeviceService deviceService;
private final DeviceProfileService deviceProfileService;
private final AttributesService attributesService;
private final DeviceCredentialsService deviceCredentialsService;
private final RuleChainService ruleChainService;
private final TimeseriesService tsService;
private final DeviceConnectivityConfiguration connectivityConfiguration;
private final QueueService queueService;
private final JwtSettingsService jwtSettingsService;
private final NotificationSettingsService notificationSettingsService;
private final NotificationTargetService notificationTargetService;
@Autowired
private BCryptPasswordEncoder passwordEncoder;
@Autowired
private UserService userService;
@Autowired
private AdminSettingsService adminSettingsService;
@Autowired
private WidgetTypeService widgetTypeService;
@Autowired
private WidgetsBundleService widgetsBundleService;
@Autowired
private TenantService tenantService;
@Autowired
private TenantProfileService tenantProfileService;
@Autowired
private CustomerService customerService;
@Autowired
private DeviceService deviceService;
@Autowired
private DeviceProfileService deviceProfileService;
@Autowired
private AttributesService attributesService;
@Autowired
private DeviceCredentialsService deviceCredentialsService;
@Autowired
private RuleChainService ruleChainService;
@Autowired
private TimeseriesService tsService;
@Autowired
private DeviceConnectivityConfiguration connectivityConfiguration;
@Value("${state.persistToTelemetry:false}")
@Getter
private boolean persistActivityToTelemetry;
@Lazy
@Autowired
private QueueService queueService;
@Autowired
private JwtSettingsService jwtSettingsService;
@Autowired
private NotificationSettingsService notificationSettingsService;
@Autowired
private NotificationTargetService notificationTargetService;
@Value("${security.jwt.tokenExpirationTime:9000}")
private Integer tokenExpirationTime;
@Value("${security.jwt.refreshTokenExpTime:604800}")
private Integer refreshTokenExpTime;
@Value("${security.jwt.tokenIssuer:thingsboard.io}")
private String tokenIssuer;
@Value("${security.jwt.tokenSigningKey:thingsboardDefaultSigningKey}")
private String tokenSigningKey;
@Bean
protected BCryptPasswordEncoder passwordEncoder() {
@ -295,7 +269,42 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
@Override
public void createRandomJwtSettings() throws Exception {
jwtSettingsService.createRandomJwtSettings();
if (jwtSettingsService.getJwtSettings() == null) {
log.info("Creating JWT admin settings...");
var jwtSettings = new JwtSettings(this.tokenExpirationTime, this.refreshTokenExpTime, this.tokenIssuer, this.tokenSigningKey);
if (isSigningKeyDefault(jwtSettings) || !validateTokenSigningKeyLength(jwtSettings)) {
jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(
RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8)));
}
jwtSettingsService.saveJwtSettings(jwtSettings);
} else {
log.info("Skip creating JWT admin settings because they already exist.");
}
}
@Override
public void updateJwtSettings() {
JwtSettings jwtSettings = jwtSettingsService.getJwtSettings();
boolean invalidSignKey = false;
String warningMessage = null;
if (isSigningKeyDefault(jwtSettings)) {
warningMessage = "The platform is using the default JWT Signing Key, which is a security risk.";
invalidSignKey = true;
} else if (!validateTokenSigningKeyLength(jwtSettings)) {
warningMessage = "The JWT Signing Key is shorter than 512 bits, which is a security risk.";
invalidSignKey = true;
}
if (invalidSignKey) {
log.warn("WARNING: {}. A new JWT Signing Key has been added automatically. " +
"You can change the JWT Signing Key using the Web UI: " +
"Navigate to \"System settings -> Security settings\" while logged in as a System Administrator.", warningMessage);
jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(
RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8)));
jwtSettingsService.saveJwtSettings(jwtSettings);
}
}
@Override

2
application/src/main/java/org/thingsboard/server/service/install/EntityDatabaseSchemaService.java

@ -21,4 +21,6 @@ public interface EntityDatabaseSchemaService extends DatabaseSchemaService {
void createOrUpdateViewsAndFunctions() throws Exception;
void createCustomerTitleUniqueConstraintIfNotExists();
}

10
application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java

@ -84,13 +84,17 @@ public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchema
}
protected void executeQuery(String query) {
executeQuery(query, null);
}
protected void executeQuery(String query, String logQuery) {
logQuery = logQuery != null ? logQuery : query;
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
log.info("Successfully executed query: {}", query);
log.info("Successfully executed query: {}", logQuery);
Thread.sleep(5000);
} catch (InterruptedException | SQLException e) {
log.error("Failed to execute query: {} due to: {}", query, e.getMessage());
throw new RuntimeException("Failed to execute query: " + query, e);
throw new RuntimeException("Failed to execute query: " + logQuery, e);
}
}

7
application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java

@ -52,4 +52,11 @@ public class SqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaSer
log.info("Installing SQL DataBase schema views and functions: " + SCHEMA_VIEWS_AND_FUNCTIONS_SQL);
executeQueryFromFile(SCHEMA_VIEWS_AND_FUNCTIONS_SQL);
}
@Override
public void createCustomerTitleUniqueConstraintIfNotExists() {
executeQuery("DO $$ BEGIN IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'customer_title_unq_key') THEN " +
"ALTER TABLE customer ADD CONSTRAINT customer_title_unq_key UNIQUE(tenant_id, title); END IF; END; $$;",
"create 'customer_title_unq_key' constraint if it doesn't already exist!");
}
}

2
application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java

@ -25,6 +25,8 @@ public interface SystemDataLoaderService {
void createRandomJwtSettings() throws Exception;
void updateJwtSettings() throws Exception;
void createOAuth2Templates() throws Exception;
void loadSystemWidgets() throws Exception;

1
application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java

@ -27,7 +27,6 @@ import org.springframework.stereotype.Service;
import java.util.Objects;
import java.util.Optional;
import static org.thingsboard.server.common.data.CacheConstants.ATTRIBUTES_CACHE;
import static org.thingsboard.server.common.data.CacheConstants.RESOURCE_INFO_CACHE;
import static org.thingsboard.server.common.data.CacheConstants.SECURITY_SETTINGS_CACHE;

82
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@ -26,22 +26,27 @@ import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.query.DynamicValue;
import org.thingsboard.server.common.data.query.FilterPredicateValue;
import org.thingsboard.server.dao.customer.CustomerDao;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.device.DeviceConnectivityConfiguration;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.sql.JpaExecutorService;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.component.RuleNodeClassInfo;
import org.thingsboard.server.utils.TbNodeUpgradeUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
@Service
@ -67,6 +72,15 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired
DeviceConnectivityConfiguration connectivityConfiguration;
@Autowired
private CustomerDao customerDao;
@Autowired
private CustomerService customerService;
@Autowired
private TenantProfileService tenantProfileService;
@Override
public void updateData(String fromVersion) throws Exception {
switch (fromVersion) {
@ -74,11 +88,79 @@ public class DefaultDataUpdateService implements DataUpdateService {
log.info("Updating data from version 3.6.0 to 3.6.1 ...");
migrateDeviceConnectivity();
break;
case "3.6.4":
log.info("Updating data from version 3.6.4 to 3.7.0 ...");
updateCustomersWithTheSameTitle();
updateMaxRuleNodeExecsPerMessage();
break;
default:
throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion);
}
}
private void updateMaxRuleNodeExecsPerMessage() {
var tenantProfiles = new PageDataIterable<>(
link -> tenantProfileService.findTenantProfiles(TenantId.SYS_TENANT_ID, link), DEFAULT_PAGE_SIZE);
tenantProfiles.forEach(tenantProfile -> {
var configurationOpt = tenantProfile.getProfileConfiguration();
configurationOpt.ifPresent(configuration -> {
if (configuration.getMaxRuleNodeExecsPerMessage() == 0) {
configuration.setMaxRuleNodeExecutionsPerMessage(1000);
try {
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile);
} catch (Exception e) {
log.error("Failed to update tenant profile with id: {} due to: ", tenantProfile.getId(), e);
}
}
});
});
}
private void updateCustomersWithTheSameTitle() {
var customers = new ArrayList<Customer>();
new PageDataIterable<>(pageLink ->
customerDao.findCustomersWithTheSameTitle(pageLink), DEFAULT_PAGE_SIZE
).forEach(customers::add);
if (customers.isEmpty()) {
return;
}
var firstCustomer = customers.get(0);
var titleToDeduplicate = firstCustomer.getTitle();
var tenantIdToDeduplicate = firstCustomer.getTenantId();
int duplicateCounter = 1;
for (int i = 1; i < customers.size(); i++) {
var currentCustomer = customers.get(i);
if (currentCustomer.getTitle().equals(titleToDeduplicate) && currentCustomer.getTenantId().equals(tenantIdToDeduplicate)) {
duplicateCounter++;
String currentTitle = currentCustomer.getTitle();
String newTitle = currentTitle + " " + duplicateCounter;
try {
Optional<Customer> customerOpt = customerService.findCustomerByTenantIdAndTitle(tenantIdToDeduplicate, newTitle);
if (customerOpt.isPresent()) {
// fallback logic: customer with title 'currentTitle + " " + duplicateCounter;' might be another duplicate.
newTitle = currentTitle + "_" + currentCustomer.getId();
}
} catch (Exception e) {
log.trace("Failed to find customer with title due to: ", e);
// fallback logic: customer with title 'currentTitle + " " + duplicateCounter;' might be another duplicate.
newTitle = currentTitle + "_" + currentCustomer.getId();
}
currentCustomer.setTitle(newTitle);
try {
customerService.saveCustomer(currentCustomer);
} catch (Exception e) {
log.error("[{}] Failed to update customer with id and title: {}, oldTitle: {}, due to: ",
currentCustomer.getTenantId(), newTitle, currentTitle, e);
}
continue;
}
titleToDeduplicate = currentCustomer.getTitle();
tenantIdToDeduplicate = currentCustomer.getTenantId();
duplicateCounter = 1;
}
}
private void migrateDeviceConnectivity() {
if (adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "connectivity") == null) {
AdminSettings connectivitySettings = new AdminSettings();

43
application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java

@ -16,12 +16,13 @@
package org.thingsboard.server.service.mail;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.util.concurrent.Futures;
import freemarker.template.Configuration;
import freemarker.template.Template;
import jakarta.xml.bind.DatatypeConverter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.MessageSource;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.NestedRuntimeException;
@ -31,29 +32,36 @@ import org.springframework.mail.javamail.JavaMailSenderImpl;
import org.springframework.mail.javamail.MimeMessageHelper;
import org.springframework.stereotype.Service;
import org.springframework.ui.freemarker.FreeMarkerTemplateUtils;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.TbEmail;
import org.thingsboard.server.cache.limits.RateLimitService;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.ApiFeature;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.ApiUsageRecordState;
import org.thingsboard.server.common.data.ApiUsageStateValue;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.exception.RateLimitExceededException;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.limit.LimitedApi;
import org.thingsboard.server.common.stats.TbApiUsageReportClient;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import jakarta.mail.internet.MimeMessage;
import java.io.ByteArrayInputStream;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@ -76,13 +84,21 @@ public class DefaultMailService implements MailService {
private TbApiUsageStateService apiUsageStateService;
@Autowired
private MailExecutorService mailExecutorService;
private MailSenderInternalExecutorService mailExecutorService;
@Autowired
private PasswordResetExecutorService passwordResetExecutorService;
@Autowired
private TbMailContextComponent tbMailContextComponent;
private TbMailContextComponent ctx;
@Autowired
private RateLimitService rateLimitService;
@Value("${mail.per_tenant_rate_limits:}")
private String perTenantRateLimitConfig;
private final ScheduledExecutorService timeoutScheduler;
private TbMailSender mailSender;
@ -95,6 +111,7 @@ public class DefaultMailService implements MailService {
this.freemarkerConfig = freemarkerConfig;
this.adminSettingsService = adminSettingsService;
this.apiUsageClient = apiUsageClient;
this.timeoutScheduler = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("mail-service-watchdog"));
}
@PostConstruct
@ -102,12 +119,19 @@ public class DefaultMailService implements MailService {
updateMailConfiguration();
}
@PreDestroy
public void destroy() {
if (timeoutScheduler != null) {
timeoutScheduler.shutdownNow();
}
}
@Override
public void updateMailConfiguration() {
AdminSettings settings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "mail");
if (settings != null) {
JsonNode jsonConfig = settings.getJsonValue();
mailSender = new TbMailSender(tbMailContextComponent, jsonConfig);
mailSender = new TbMailSender(ctx, jsonConfig);
mailFrom = jsonConfig.get("mailFrom").asText();
timeout = jsonConfig.get("timeout").asLong(DEFAULT_TIMEOUT);
} else {
@ -122,7 +146,7 @@ public class DefaultMailService implements MailService {
@Override
public void sendTestMail(JsonNode jsonConfig, String email) throws ThingsboardException {
TbMailSender testMailSender = new TbMailSender(tbMailContextComponent, jsonConfig);
TbMailSender testMailSender = new TbMailSender(ctx, jsonConfig);
String mailFrom = jsonConfig.get("mailFrom").asText();
String subject = messages.getMessage("test.message.subject", null, Locale.US);
long timeout = jsonConfig.get("timeout").asLong(DEFAULT_TIMEOUT);
@ -214,6 +238,10 @@ public class DefaultMailService implements MailService {
private void sendMail(TenantId tenantId, CustomerId customerId, TbEmail tbEmail, JavaMailSender javaMailSender, long timeout) throws ThingsboardException {
if (apiUsageStateService.getApiUsageState(tenantId).isEmailSendEnabled()) {
if (tenantId != null && !tenantId.isSysTenantId() && StringUtils.isNotEmpty(perTenantRateLimitConfig) &&
!rateLimitService.checkRateLimit(LimitedApi.EMAILS, (Object) tenantId, perTenantRateLimitConfig)) {
throw new RateLimitExceededException(LimitedApi.EMAILS);
}
try {
MimeMessage mailMsg = javaMailSender.createMimeMessage();
boolean multipart = (tbEmail.getImages() != null && !tbEmail.getImages().isEmpty());
@ -415,8 +443,11 @@ public class DefaultMailService implements MailService {
}
private void sendMailWithTimeout(JavaMailSender mailSender, MimeMessage msg, long timeout) {
var submittedMail = Futures.withTimeout(
mailExecutorService.submit(() -> mailSender.send(msg)),
timeout, TimeUnit.MILLISECONDS, timeoutScheduler);
try {
mailExecutorService.submit(() -> mailSender.send(msg)).get(timeout, TimeUnit.MILLISECONDS);
submittedMail.get(timeout, TimeUnit.MILLISECONDS);
} catch (TimeoutException e) {
log.debug("Error during mail submission", e);
throw new RuntimeException("Timeout!");

37
application/src/main/java/org/thingsboard/server/service/mail/MailSenderInternalExecutorService.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.mail;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.common.util.AbstractListeningExecutor;
/**
* Executor have the sole purpose to send mails. It should be used only by Mail Service.
* For other purposes please use the MailExecutorService component
* */
@Component
public class MailSenderInternalExecutorService extends AbstractListeningExecutor {
@Value("${actors.rule.mail_thread_pool_size}")
private int mailExecutorThreadPoolSize;
@Override
protected int getThreadPollSize() {
return mailExecutorThreadPoolSize;
}
}

1
application/src/main/java/org/thingsboard/server/service/mail/TbMailContextComponent.java

@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
@Component
@Data

3
application/src/main/java/org/thingsboard/server/service/notification/channels/MobileAppNotificationChannel.java

@ -105,9 +105,10 @@ public class MobileAppNotificationChannel implements NotificationChannel<User, M
firebaseService.sendMessage(ctx.getTenantId(), credentials, token, subject, body, data, unreadCount);
} catch (FirebaseMessagingException e) {
MessagingErrorCode errorCode = e.getMessagingErrorCode();
if (errorCode == MessagingErrorCode.UNREGISTERED || errorCode == MessagingErrorCode.INVALID_ARGUMENT) {
if (errorCode == MessagingErrorCode.UNREGISTERED || errorCode == MessagingErrorCode.INVALID_ARGUMENT || errorCode == MessagingErrorCode.SENDER_ID_MISMATCH) {
validTokens.remove(token);
userService.removeMobileSession(recipient.getTenantId(), token);
log.debug("[{}][{}] Removed invalid FCM token due to {} {} ({})", recipient.getTenantId(), recipient.getId(), errorCode, e.getMessage(), token);
continue;
}
throw new RuntimeException("Failed to send message via FCM: " + e.getMessage(), e);

416
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -20,12 +20,12 @@ import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.id.NotificationRequestId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTrigger;
import org.thingsboard.server.common.data.queue.QueueConfig;
import org.thingsboard.server.common.data.rpc.RpcError;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg;
@ -78,10 +79,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM
import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.edge.EdgeNotificationService;
@ -89,6 +91,7 @@ import org.thingsboard.server.service.notification.NotificationSchedulerService;
import org.thingsboard.server.service.ota.OtaPackageStateService;
import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
import org.thingsboard.server.service.queue.processing.IdMsgPair;
import org.thingsboard.server.service.resource.TbImageService;
@ -109,7 +112,6 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@ -122,9 +124,11 @@ import java.util.stream.Collectors;
public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCoreNotificationMsg> implements TbCoreConsumerService {
@Value("${queue.core.poll-interval}")
private long pollDuration;
private long pollInterval;
@Value("${queue.core.pack-processing-timeout}")
private long packProcessingTimeout;
@Value("${queue.core.consumer-per-partition:true}")
private boolean consumerPerPartition;
@Value("${queue.core.stats.enabled:false}")
private boolean statsEnabled;
@ -133,7 +137,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
@Value("${queue.core.ota.pack-size:100}")
private int firmwarePackSize;
private final TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> mainConsumer;
private final DeviceStateService stateService;
private final TbApiUsageStateService statsService;
private final TbLocalSubscriptionService localSubscriptionService;
@ -144,14 +147,14 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
private final GitVersionControlQueueService vcQueueService;
private final NotificationSchedulerService notificationSchedulerService;
private final NotificationRuleProcessor notificationRuleProcessor;
private final TbCoreConsumerStats stats;
protected final TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
private final TbQueueConsumer<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> firmwareStatesConsumer;
private final TbCoreQueueFactory queueFactory;
private final TbImageService imageService;
private final TbCoreConsumerStats stats;
private MainQueueConsumerManager<TbProtoQueueMsg<ToCoreMsg>, CoreQueueConfig> mainConsumer;
private QueueConsumerManager<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
private QueueConsumerManager<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> firmwareStatesConsumer;
protected volatile ExecutorService consumersExecutor;
protected volatile ExecutorService usageStatsExecutor;
private volatile ExecutorService firmwareStatesExecutor;
private volatile ListeningExecutorService deviceActivityEventsExecutor;
public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory,
@ -175,10 +178,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
NotificationSchedulerService notificationSchedulerService,
NotificationRuleProcessor notificationRuleProcessor,
TbImageService imageService) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer(), jwtSettingsService);
this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer();
this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer();
this.firmwareStatesConsumer = tbCoreQueueFactory.createToOtaPackageStateServiceMsgConsumer();
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService,
eventPublisher, jwtSettingsService);
this.stateService = stateService;
this.localSubscriptionService = localSubscriptionService;
this.subscriptionManagerService = subscriptionManagerService;
@ -191,152 +192,146 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
this.notificationSchedulerService = notificationSchedulerService;
this.notificationRuleProcessor = notificationRuleProcessor;
this.imageService = imageService;
this.queueFactory = tbCoreQueueFactory;
}
@PostConstruct
public void init() {
super.init("tb-core-notifications-consumer");
this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-consumer"));
this.usageStatsExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer"));
this.firmwareStatesExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-firmware-notifications-consumer"));
super.init("tb-core");
this.deviceActivityEventsExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-device-activity-events-executor")));
this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToCoreMsg>, CoreQueueConfig>builder()
.queueKey(new QueueKey(ServiceType.TB_CORE))
.config(CoreQueueConfig.of(consumerPerPartition, (int) pollInterval))
.msgPackProcessor(this::processMsgs)
.consumerCreator(config -> queueFactory.createToCoreMsgConsumer())
.consumerExecutor(consumersExecutor)
.scheduler(scheduler)
.taskExecutor(mgmtExecutor)
.build();
this.usageStatsConsumer = QueueConsumerManager.<TbProtoQueueMsg<ToUsageStatsServiceMsg>>builder()
.name("TB Usage Stats")
.msgPackProcessor(this::processUsageStatsMsg)
.pollInterval(pollInterval)
.consumerCreator(queueFactory::createToUsageStatsServiceMsgConsumer)
.consumerExecutor(consumersExecutor)
.threadPrefix("usage-stats")
.build();
this.firmwareStatesConsumer = QueueConsumerManager.<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>>builder()
.name("TB Ota Package States")
.msgPackProcessor(this::processFirmwareMsgs)
.pollInterval(pollInterval)
.consumerCreator(queueFactory::createToOtaPackageStateServiceMsgConsumer)
.consumerExecutor(consumersExecutor)
.threadPrefix("firmware")
.build();
}
@PreDestroy
public void destroy() {
super.destroy();
if (consumersExecutor != null) {
consumersExecutor.shutdownNow();
}
if (usageStatsExecutor != null) {
usageStatsExecutor.shutdownNow();
}
if (firmwareStatesExecutor != null) {
firmwareStatesExecutor.shutdownNow();
}
if (deviceActivityEventsExecutor != null) {
deviceActivityEventsExecutor.shutdownNow();
}
}
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void onApplicationEvent(ApplicationReadyEvent event) {
super.onApplicationEvent(event);
launchUsageStatsConsumer();
launchOtaPackageUpdateNotificationConsumer();
@Override
protected void startConsumers() {
super.startConsumers();
firmwareStatesConsumer.subscribe();
firmwareStatesConsumer.launch();
usageStatsConsumer.launch();
}
@Override
protected void onTbApplicationEvent(PartitionChangeEvent event) {
log.info("Subscribing to partitions: {}", event.getPartitions());
this.mainConsumer.subscribe(event.getPartitions());
this.usageStatsConsumer.subscribe(
event
.getPartitions()
.stream()
.map(tpi -> tpi.newByTopic(usageStatsConsumer.getTopic()))
.collect(Collectors.toSet()));
this.firmwareStatesConsumer.subscribe();
}
@Override
protected void launchMainConsumers() {
consumersExecutor.submit(() -> {
while (!stopped) {
mainConsumer.update(event.getPartitions());
usageStatsConsumer.subscribe(event.getPartitions()
.stream()
.map(tpi -> tpi.newByTopic(usageStatsConsumer.getConsumer().getTopic()))
.collect(Collectors.toSet()));
}
private void processMsgs(List<TbProtoQueueMsg<ToCoreMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> consumer, CoreQueueConfig config) throws Exception {
List<IdMsgPair<ToCoreMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).collect(Collectors.toList());
ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<ToCoreMsg>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
PendingMsgHolder pendingMsgHolder = new PendingMsgHolder();
Future<?> packSubmitFuture = consumersExecutor.submit(() -> {
orderedMsgList.forEach((element) -> {
UUID id = element.getUuid();
TbProtoQueueMsg<ToCoreMsg> msg = element.getMsg();
log.trace("[{}] Creating main callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
List<TbProtoQueueMsg<ToCoreMsg>> msgs = mainConsumer.poll(pollDuration);
if (msgs.isEmpty()) {
continue;
}
List<IdMsgPair<ToCoreMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).collect(Collectors.toList());
ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<ToCoreMsg>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
PendingMsgHolder pendingMsgHolder = new PendingMsgHolder();
Future<?> packSubmitFuture = consumersExecutor.submit(() -> {
orderedMsgList.forEach((element) -> {
UUID id = element.getUuid();
TbProtoQueueMsg<ToCoreMsg> msg = element.getMsg();
log.trace("[{}] Creating main callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
ToCoreMsg toCoreMsg = msg.getValue();
pendingMsgHolder.setToCoreMsg(toCoreMsg);
if (toCoreMsg.hasToSubscriptionMgrMsg()) {
log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg());
forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback);
} else if (toCoreMsg.hasToDeviceActorMsg()) {
log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg());
forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback);
} else if (toCoreMsg.hasDeviceStateServiceMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceStateServiceMsg());
forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback);
} else if (toCoreMsg.hasEdgeNotificationMsg()) {
log.trace("[{}] Forwarding message to edge service {}", id, toCoreMsg.getEdgeNotificationMsg());
forwardToEdgeNotificationService(toCoreMsg.getEdgeNotificationMsg(), callback);
} else if (toCoreMsg.hasDeviceConnectMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceConnectMsg());
forwardToStateService(toCoreMsg.getDeviceConnectMsg(), callback);
} else if (toCoreMsg.hasDeviceActivityMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceActivityMsg());
forwardToStateService(toCoreMsg.getDeviceActivityMsg(), callback);
} else if (toCoreMsg.hasDeviceDisconnectMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceDisconnectMsg());
forwardToStateService(toCoreMsg.getDeviceDisconnectMsg(), callback);
} else if (toCoreMsg.hasDeviceInactivityMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceInactivityMsg());
forwardToStateService(toCoreMsg.getDeviceInactivityMsg(), callback);
} else if (toCoreMsg.hasToDeviceActorNotification()) {
TbActorMsg actorMsg = ProtoUtils.fromProto(toCoreMsg.getToDeviceActorNotification());
if (actorMsg != null) {
if (actorMsg.getMsgType().equals(MsgType.DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG)) {
tbCoreDeviceRpcService.forwardRpcRequestToDeviceActor((ToDeviceRpcRequestActorMsg) actorMsg);
} else {
log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg);
actorContext.tell(actorMsg);
}
}
callback.onSuccess();
} else if (toCoreMsg.hasNotificationSchedulerServiceMsg()) {
TransportProtos.NotificationSchedulerServiceMsg notificationSchedulerServiceMsg = toCoreMsg.getNotificationSchedulerServiceMsg();
log.trace("[{}] Forwarding message to notification scheduler service {}", id, toCoreMsg.getNotificationSchedulerServiceMsg());
forwardToNotificationSchedulerService(notificationSchedulerServiceMsg, callback);
} else if (toCoreMsg.hasErrorEventMsg()) {
forwardToEventService(toCoreMsg.getErrorEventMsg(), callback);
} else if (toCoreMsg.hasLifecycleEventMsg()) {
forwardToEventService(toCoreMsg.getLifecycleEventMsg(), callback);
}
} catch (Throwable e) {
log.warn("[{}] Failed to process message: {}", id, msg, e);
callback.onFailure(e);
ToCoreMsg toCoreMsg = msg.getValue();
pendingMsgHolder.setToCoreMsg(toCoreMsg);
if (toCoreMsg.hasToSubscriptionMgrMsg()) {
log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg());
forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback);
} else if (toCoreMsg.hasToDeviceActorMsg()) {
log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg());
forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback);
} else if (toCoreMsg.hasDeviceStateServiceMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceStateServiceMsg());
forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback);
} else if (toCoreMsg.hasEdgeNotificationMsg()) {
log.trace("[{}] Forwarding message to edge service {}", id, toCoreMsg.getEdgeNotificationMsg());
forwardToEdgeNotificationService(toCoreMsg.getEdgeNotificationMsg(), callback);
} else if (toCoreMsg.hasDeviceConnectMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceConnectMsg());
forwardToStateService(toCoreMsg.getDeviceConnectMsg(), callback);
} else if (toCoreMsg.hasDeviceActivityMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceActivityMsg());
forwardToStateService(toCoreMsg.getDeviceActivityMsg(), callback);
} else if (toCoreMsg.hasDeviceDisconnectMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceDisconnectMsg());
forwardToStateService(toCoreMsg.getDeviceDisconnectMsg(), callback);
} else if (toCoreMsg.hasDeviceInactivityMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceInactivityMsg());
forwardToStateService(toCoreMsg.getDeviceInactivityMsg(), callback);
} else if (toCoreMsg.hasToDeviceActorNotification()) {
TbActorMsg actorMsg = ProtoUtils.fromProto(toCoreMsg.getToDeviceActorNotification());
if (actorMsg != null) {
if (actorMsg.getMsgType().equals(MsgType.DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG)) {
tbCoreDeviceRpcService.forwardRpcRequestToDeviceActor((ToDeviceRpcRequestActorMsg) actorMsg);
} else {
log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg);
actorContext.tell(actorMsg);
}
});
});
if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) {
if (!packSubmitFuture.isDone()) {
packSubmitFuture.cancel(true);
ToCoreMsg lastSubmitMsg = pendingMsgHolder.getToCoreMsg();
log.info("Timeout to process message: {}", lastSubmitMsg);
}
ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue()));
}
mainConsumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to obtain messages from queue.", e);
try {
Thread.sleep(pollDuration);
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new requests", e2);
}
callback.onSuccess();
} else if (toCoreMsg.hasNotificationSchedulerServiceMsg()) {
TransportProtos.NotificationSchedulerServiceMsg notificationSchedulerServiceMsg = toCoreMsg.getNotificationSchedulerServiceMsg();
log.trace("[{}] Forwarding message to notification scheduler service {}", id, toCoreMsg.getNotificationSchedulerServiceMsg());
forwardToNotificationSchedulerService(notificationSchedulerServiceMsg, callback);
} else if (toCoreMsg.hasErrorEventMsg()) {
forwardToEventService(toCoreMsg.getErrorEventMsg(), callback);
} else if (toCoreMsg.hasLifecycleEventMsg()) {
forwardToEventService(toCoreMsg.getLifecycleEventMsg(), callback);
}
} catch (Throwable e) {
log.warn("[{}] Failed to process message: {}", id, msg, e);
callback.onFailure(e);
}
}
log.info("TB Core Consumer stopped.");
});
});
if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) {
if (!packSubmitFuture.isDone()) {
packSubmitFuture.cancel(true);
ToCoreMsg lastSubmitMsg = pendingMsgHolder.getToCoreMsg();
log.info("Timeout to process message: {}", lastSubmitMsg);
}
if (log.isDebugEnabled()) {
ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue()));
}
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue()));
}
consumer.commit();
}
private static class PendingMsgHolder {
@ -352,7 +347,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
@Override
protected long getNotificationPollDuration() {
return pollDuration;
return pollInterval;
}
@Override
@ -360,6 +355,16 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
return packProcessingTimeout;
}
@Override
protected int getMgmtThreadPoolSize() {
return Math.max(Runtime.getRuntime().availableProcessors(), 4);
}
@Override
protected TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createNotificationsConsumer() {
return queueFactory.createToCoreNotificationsMsgConsumer();
}
@Override
protected void handleNotification(UUID id, TbProtoQueueMsg<ToCoreNotificationMsg> msg, TbCallback callback) {
ToCoreNotificationMsg toCoreNotification = msg.getValue();
@ -408,92 +413,55 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
}
}
private void launchUsageStatsConsumer() {
usageStatsExecutor.submit(() -> {
while (!stopped) {
try {
List<TbProtoQueueMsg<ToUsageStatsServiceMsg>> msgs = usageStatsConsumer.poll(getNotificationPollDuration());
if (msgs.isEmpty()) {
continue;
}
ConcurrentMap<UUID, TbProtoQueueMsg<ToUsageStatsServiceMsg>> pendingMap = msgs.stream().collect(
Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<ToUsageStatsServiceMsg>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
pendingMap.forEach((id, msg) -> {
log.trace("[{}] Creating usage stats callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
handleUsageStats(msg, callback);
} catch (Throwable e) {
log.warn("[{}] Failed to process usage stats: {}", id, msg, e);
callback.onFailure(e);
}
});
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process usage stats: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process usage stats: {}", id, msg.getValue()));
}
usageStatsConsumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to obtain usage stats from queue.", e);
try {
Thread.sleep(getNotificationPollDuration());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new usage stats", e2);
}
}
}
private void processUsageStatsMsg(List<TbProtoQueueMsg<ToUsageStatsServiceMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> consumer) throws Exception {
ConcurrentMap<UUID, TbProtoQueueMsg<ToUsageStatsServiceMsg>> pendingMap = msgs.stream().collect(
Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<ToUsageStatsServiceMsg>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
pendingMap.forEach((id, msg) -> {
log.trace("[{}] Creating usage stats callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
handleUsageStats(msg, callback);
} catch (Throwable e) {
log.warn("[{}] Failed to process usage stats: {}", id, msg, e);
callback.onFailure(e);
}
log.info("TB Usage Stats Consumer stopped.");
});
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process usage stats: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process usage stats: {}", id, msg.getValue()));
}
consumer.commit();
}
private void launchOtaPackageUpdateNotificationConsumer() {
private void processFirmwareMsgs(List<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> consumer) {
long maxProcessingTimeoutPerRecord = firmwarePackInterval / firmwarePackSize;
firmwareStatesExecutor.submit(() -> {
while (!stopped) {
try {
List<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> msgs = firmwareStatesConsumer.poll(getNotificationPollDuration());
if (msgs.isEmpty()) {
continue;
}
long timeToSleep = maxProcessingTimeoutPerRecord;
for (TbProtoQueueMsg<ToOtaPackageStateServiceMsg> msg : msgs) {
try {
long startTime = System.currentTimeMillis();
boolean isSuccessUpdate = handleOtaPackageUpdates(msg);
long endTime = System.currentTimeMillis();
long spentTime = endTime - startTime;
timeToSleep = timeToSleep - spentTime;
if (isSuccessUpdate) {
if (timeToSleep > 0) {
log.debug("Spent time per record is: [{}]!", spentTime);
Thread.sleep(timeToSleep);
timeToSleep = 0;
}
timeToSleep += maxProcessingTimeoutPerRecord;
}
} catch (Throwable e) {
log.warn("Failed to process firmware update msg: {}", msg, e);
}
}
firmwareStatesConsumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to obtain usage stats from queue.", e);
try {
Thread.sleep(getNotificationPollDuration());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new firmware updates", e2);
}
long timeToSleep = maxProcessingTimeoutPerRecord;
for (TbProtoQueueMsg<ToOtaPackageStateServiceMsg> msg : msgs) {
try {
long startTime = System.currentTimeMillis();
boolean isSuccessUpdate = handleOtaPackageUpdates(msg);
long endTime = System.currentTimeMillis();
long spentTime = endTime - startTime;
timeToSleep = timeToSleep - spentTime;
if (isSuccessUpdate) {
if (timeToSleep > 0) {
log.debug("Spent time per record is: [{}]!", spentTime);
Thread.sleep(timeToSleep);
timeToSleep = 0;
}
timeToSleep += maxProcessingTimeoutPerRecord;
}
} catch (InterruptedException e) {
return;
} catch (Throwable e) {
log.warn("Failed to process firmware update msg: {}", msg, e);
}
log.info("TB Ota Package States Consumer stopped.");
});
}
consumer.commit();
}
private void handleUsageStats(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
@ -781,15 +749,17 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
@Override
protected void stopConsumers() {
if (mainConsumer != null) {
mainConsumer.unsubscribe();
}
if (usageStatsConsumer != null) {
usageStatsConsumer.unsubscribe();
}
if (firmwareStatesConsumer != null) {
firmwareStatesConsumer.unsubscribe();
}
super.stopConsumers();
mainConsumer.stop();
mainConsumer.awaitStop();
usageStatsConsumer.stop();
firmwareStatesConsumer.stop();
}
@Data(staticConstructor = "of")
public static class CoreQueueConfig implements QueueConfig {
private final boolean consumerPerPartition;
private final int pollInterval;
}
}

41
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -15,8 +15,8 @@
*/
package org.thingsboard.server.service.queue;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled;
@ -39,12 +39,11 @@ import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.QueueDeleteMsg;
import org.thingsboard.server.gen.transport.TransportProtos.QueueUpdateMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.TbRuleEngineComponent;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.profile.TbAssetProfileCache;
@ -55,7 +54,6 @@ import org.thingsboard.server.service.queue.ruleengine.TbRuleEngineQueueConsumer
import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService;
import jakarta.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@ -76,7 +74,6 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
private final ConcurrentMap<QueueKey, TbRuleEngineQueueConsumerManager> consumers = new ConcurrentHashMap<>();
public DefaultTbRuleEngineConsumerService(TbRuleEngineConsumerContext ctx,
TbRuleEngineQueueFactory tbRuleEngineQueueFactory,
ActorSystemContext actorContext,
TbRuleEngineDeviceRpcService tbDeviceRpcService,
QueueService queueService,
@ -87,8 +84,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
PartitionService partitionService,
ApplicationEventPublisher eventPublisher,
JwtSettingsService jwtSettingsService) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService,
eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), jwtSettingsService);
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService);
this.ctx = ctx;
this.tbDeviceRpcService = tbDeviceRpcService;
this.queueService = queueService;
@ -96,7 +92,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
@PostConstruct
public void init() {
super.init("tb-rule-engine-notifications-consumer");
super.init("tb-rule-engine");
List<Queue> queues = queueService.findAllQueues();
for (Queue configuration : queues) {
if (partitionService.isManagedByCurrentService(configuration.getTenantId())) {
@ -136,20 +132,11 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
});
}
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void onApplicationEvent(ApplicationReadyEvent event) {
super.onApplicationEvent(event);
ctx.setReady(true);
}
@Override
protected void launchMainConsumers() {}
@Override
protected void stopConsumers() {
super.stopConsumers();
consumers.values().forEach(TbRuleEngineQueueConsumerManager::stop);
consumers.values().forEach(TbRuleEngineQueueConsumerManager::awaitStop);
ctx.stop();
}
@Override
@ -167,6 +154,16 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
return ctx.getPackProcessingTimeout();
}
@Override
protected int getMgmtThreadPoolSize() {
return ctx.getMgmtThreadPoolSize();
}
@Override
protected TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createNotificationsConsumer() {
return ctx.getQueueFactory().createToRuleEngineNotificationsMsgConsumer();
}
@Override
protected void handleNotification(UUID id, TbProtoQueueMsg<ToRuleEngineNotificationMsg> msg, TbCallback callback) throws Exception {
ToRuleEngineNotificationMsg nfMsg = msg.getValue();
@ -243,7 +240,13 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
}
private TbRuleEngineQueueConsumerManager createConsumer(QueueKey queueKey, Queue queue) {
var consumer = new TbRuleEngineQueueConsumerManager(ctx, queueKey);
var consumer = TbRuleEngineQueueConsumerManager.create()
.ctx(ctx)
.queueKey(queueKey)
.consumerExecutor(consumersExecutor)
.scheduler(scheduler)
.taskExecutor(mgmtExecutor)
.build();
consumers.put(queueKey, consumer);
consumer.init(queue);
return consumer;

322
application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java

@ -0,0 +1,322 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue.consumer;
import lombok.Builder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.queue.QueueConfig;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.service.queue.ruleengine.QueueEvent;
import org.thingsboard.server.service.queue.ruleengine.TbQueueConsumerManagerTask;
import org.thingsboard.server.service.queue.ruleengine.TbQueueConsumerTask;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
public class MainQueueConsumerManager<M extends TbQueueMsg, C extends QueueConfig> {
protected final QueueKey queueKey;
@Getter
protected C config;
protected final MsgPackProcessor<M, C> msgPackProcessor;
protected final Function<C, TbQueueConsumer<M>> consumerCreator;
protected final ExecutorService consumerExecutor;
protected final ScheduledExecutorService scheduler;
protected final ExecutorService taskExecutor;
private final java.util.Queue<TbQueueConsumerManagerTask> tasks = new ConcurrentLinkedQueue<>();
private final ReentrantLock lock = new ReentrantLock();
@Getter
private volatile Set<TopicPartitionInfo> partitions;
protected volatile ConsumerWrapper<M> consumerWrapper;
protected volatile boolean stopped;
@Builder
public MainQueueConsumerManager(QueueKey queueKey, C config,
MsgPackProcessor<M, C> msgPackProcessor,
Function<C, TbQueueConsumer<M>> consumerCreator,
ExecutorService consumerExecutor,
ScheduledExecutorService scheduler,
ExecutorService taskExecutor) {
this.queueKey = queueKey;
this.config = config;
this.msgPackProcessor = msgPackProcessor;
this.consumerCreator = consumerCreator;
this.consumerExecutor = consumerExecutor;
this.scheduler = scheduler;
this.taskExecutor = taskExecutor;
if (config != null) {
init(config);
}
}
public void init(C config) {
this.config = config;
if (config.isConsumerPerPartition()) {
this.consumerWrapper = new ConsumerPerPartitionWrapper();
} else {
this.consumerWrapper = new SingleConsumerWrapper();
}
log.debug("[{}] Initialized consumer for queue: {}", queueKey, config);
}
public void update(C config) {
addTask(TbQueueConsumerManagerTask.configUpdate(config));
}
public void update(Set<TopicPartitionInfo> partitions) {
addTask(TbQueueConsumerManagerTask.partitionChange(partitions));
}
protected void addTask(TbQueueConsumerManagerTask todo) {
if (stopped) {
return;
}
tasks.add(todo);
log.trace("[{}] Added task: {}", queueKey, todo);
tryProcessTasks();
}
private void tryProcessTasks() {
taskExecutor.submit(() -> {
if (lock.tryLock()) {
try {
C newConfig = null;
Set<TopicPartitionInfo> newPartitions = null;
while (!stopped) {
TbQueueConsumerManagerTask task = tasks.poll();
if (task == null) {
break;
}
log.trace("[{}] Processing task: {}", queueKey, task);
if (task.getEvent() == QueueEvent.PARTITION_CHANGE) {
newPartitions = task.getPartitions();
} else if (task.getEvent() == QueueEvent.CONFIG_UPDATE) {
newConfig = (C) task.getConfig();
} else {
processTask(task);
}
}
if (stopped) {
return;
}
if (newConfig != null) {
doUpdate(newConfig);
}
if (newPartitions != null) {
doUpdate(newPartitions);
}
} catch (Exception e) {
log.error("[{}] Failed to process tasks", queueKey, e);
} finally {
lock.unlock();
}
} else {
log.trace("[{}] Failed to acquire lock", queueKey);
scheduler.schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS);
}
});
}
protected void processTask(TbQueueConsumerManagerTask task) {
}
private void doUpdate(C newConfig) {
log.info("[{}] Processing queue update: {}", queueKey, newConfig);
var oldConfig = this.config;
this.config = newConfig;
if (log.isTraceEnabled()) {
log.trace("[{}] Old queue configuration: {}", queueKey, oldConfig);
log.trace("[{}] New queue configuration: {}", queueKey, newConfig);
}
if (oldConfig == null) {
init(config);
} else if (newConfig.isConsumerPerPartition() != oldConfig.isConsumerPerPartition()) {
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop);
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion);
init(config);
if (partitions != null) {
doUpdate(partitions); // even if partitions number was changed, there can be no partition change event
}
} else {
log.trace("[{}] Silently applied new config, because consumer-per-partition not changed", queueKey);
// do nothing, because partitions change (if they changed) will be handled on PartitionChangeEvent,
// and changes to other config values will be picked up by consumer on the fly,
// and queue topic and name are immutable
}
}
private void doUpdate(Set<TopicPartitionInfo> partitions) {
this.partitions = partitions;
consumerWrapper.updatePartitions(partitions);
}
private void launchConsumer(TbQueueConsumerTask<M> consumerTask) {
log.info("[{}] Launching consumer", consumerTask.getKey());
Future<?> consumerLoop = consumerExecutor.submit(() -> {
ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString());
try {
consumerLoop(consumerTask.getConsumer());
} catch (Throwable e) {
log.error("Failure in consumer loop", e);
}
log.info("[{}] Consumer stopped", consumerTask.getKey());
});
consumerTask.setTask(consumerLoop);
}
private void consumerLoop(TbQueueConsumer<M> consumer) {
while (!stopped && !consumer.isStopped()) {
try {
List<M> msgs = consumer.poll(config.getPollInterval());
if (msgs.isEmpty()) {
continue;
}
processMsgs(msgs, consumer, config);
} catch (Exception e) {
if (!consumer.isStopped()) {
log.warn("Failed to process messages from queue", e);
try {
Thread.sleep(config.getPollInterval());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new requests", e2);
}
}
}
}
if (consumer.isStopped()) {
consumer.unsubscribe();
}
}
protected void processMsgs(List<M> msgs, TbQueueConsumer<M> consumer, C config) throws Exception {
msgPackProcessor.process(msgs, consumer, config);
}
public void stop() {
log.debug("[{}] Stopping consumers", queueKey);
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop);
stopped = true;
}
public void awaitStop() {
log.debug("[{}] Waiting for consumers to stop", queueKey);
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion);
log.debug("[{}] Unsubscribed and stopped consumers", queueKey);
}
private static String partitionsToString(Collection<TopicPartitionInfo> partitions) {
return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.joining(", ", "[", "]"));
}
public interface MsgPackProcessor<M extends TbQueueMsg, C extends QueueConfig> {
void process(List<M> msgs, TbQueueConsumer<M> consumer, C config) throws Exception;
}
public interface ConsumerWrapper<M extends TbQueueMsg> {
void updatePartitions(Set<TopicPartitionInfo> partitions);
Collection<TbQueueConsumerTask<M>> getConsumers();
}
class ConsumerPerPartitionWrapper implements ConsumerWrapper<M> {
private final Map<TopicPartitionInfo, TbQueueConsumerTask<M>> consumers = new HashMap<>();
@Override
public void updatePartitions(Set<TopicPartitionInfo> partitions) {
Set<TopicPartitionInfo> addedPartitions = new HashSet<>(partitions);
addedPartitions.removeAll(consumers.keySet());
Set<TopicPartitionInfo> removedPartitions = new HashSet<>(consumers.keySet());
removedPartitions.removeAll(partitions);
log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, partitionsToString(addedPartitions), partitionsToString(removedPartitions));
removedPartitions.forEach((tpi) -> consumers.get(tpi).initiateStop());
removedPartitions.forEach((tpi) -> consumers.remove(tpi).awaitCompletion());
addedPartitions.forEach((tpi) -> {
String key = queueKey + "-" + tpi.getPartition().orElse(-1);
TbQueueConsumerTask<M> consumer = new TbQueueConsumerTask<>(key, consumerCreator.apply(config));
consumers.put(tpi, consumer);
consumer.subscribe(Set.of(tpi));
launchConsumer(consumer);
});
}
@Override
public Collection<TbQueueConsumerTask<M>> getConsumers() {
return consumers.values();
}
}
class SingleConsumerWrapper implements ConsumerWrapper<M> {
private TbQueueConsumerTask<M> consumer;
@Override
public void updatePartitions(Set<TopicPartitionInfo> partitions) {
log.info("[{}] New partitions: {}", queueKey, partitionsToString(partitions));
if (partitions.isEmpty()) {
if (consumer != null && consumer.isRunning()) {
consumer.initiateStop();
consumer.awaitCompletion();
}
consumer = null;
return;
}
if (consumer == null) {
consumer = new TbQueueConsumerTask<>(queueKey, consumerCreator.apply(config));
}
consumer.subscribe(partitions);
if (!consumer.isRunning()) {
launchConsumer(consumer);
}
}
@Override
public Collection<TbQueueConsumerTask<M>> getConsumers() {
if (consumer == null) {
return Collections.emptyList();
}
return List.of(consumer);
}
}
}

131
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@ -15,10 +15,11 @@
*/
package org.thingsboard.server.service.queue.processing;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationEventPublisher;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.EntityType;
@ -36,6 +37,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
@ -47,7 +49,6 @@ import org.thingsboard.server.service.queue.TbPackCallback;
import org.thingsboard.server.service.queue.TbPackProcessingContext;
import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService;
import jakarta.annotation.PreDestroy;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
@ -55,6 +56,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@ -62,9 +64,6 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor
public abstract class AbstractConsumerService<N extends com.google.protobuf.GeneratedMessageV3> extends TbApplicationEventListener<PartitionChangeEvent> {
protected volatile ExecutorService notificationsConsumerExecutor;
protected volatile boolean stopped = false;
protected volatile boolean isReady = false;
protected final ActorSystemContext actorContext;
protected final TbTenantProfileCache tenantProfileCache;
protected final TbDeviceProfileCache deviceProfileCache;
@ -72,21 +71,37 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
protected final TbApiUsageStateService apiUsageStateService;
protected final PartitionService partitionService;
protected final ApplicationEventPublisher eventPublisher;
protected final TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer;
protected final JwtSettingsService jwtSettingsService;
public void init(String nfConsumerThreadName) {
this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(nfConsumerThreadName));
protected QueueConsumerManager<TbProtoQueueMsg<N>> nfConsumer;
protected ExecutorService consumersExecutor;
protected ExecutorService mgmtExecutor;
protected ScheduledExecutorService scheduler;
public void init(String prefix) {
this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName(prefix + "-consumer"));
this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(getMgmtThreadPoolSize(), prefix + "-mgmt");
this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName(prefix + "-consumer-scheduler"));
this.nfConsumer = QueueConsumerManager.<TbProtoQueueMsg<N>>builder()
.name(getServiceType().getLabel() + " Notifications")
.msgPackProcessor(this::processNotifications)
.pollInterval(getNotificationPollDuration())
.consumerCreator(this::createNotificationsConsumer)
.consumerExecutor(consumersExecutor)
.threadPrefix("notifications")
.build();
}
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void onApplicationEvent(ApplicationReadyEvent event) {
log.info("Subscribing to notifications: {}", nfConsumer.getTopic());
this.nfConsumer.subscribe();
this.isReady = true;
launchNotificationsConsumer();
launchMainConsumers();
public void afterStartUp() {
startConsumers();
}
protected void startConsumers() {
nfConsumer.subscribe();
nfConsumer.launch();
}
@Override
@ -96,58 +111,42 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
protected abstract ServiceType getServiceType();
protected abstract void launchMainConsumers();
protected abstract void stopConsumers();
protected void stopConsumers() {
nfConsumer.stop();
}
protected abstract long getNotificationPollDuration();
protected abstract long getNotificationPackProcessingTimeout();
protected void launchNotificationsConsumer() {
notificationsConsumerExecutor.submit(() -> {
while (!stopped) {
try {
List<TbProtoQueueMsg<N>> msgs = nfConsumer.poll(getNotificationPollDuration());
if (msgs.isEmpty()) {
continue;
}
List<IdMsgPair<N>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).collect(Collectors.toList());
ConcurrentMap<UUID, TbProtoQueueMsg<N>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<N>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
orderedMsgList.forEach(element -> {
UUID id = element.getUuid();
TbProtoQueueMsg<N> msg = element.getMsg();
log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
handleNotification(id, msg, callback);
} catch (Throwable e) {
log.warn("[{}] Failed to process notification: {}", id, msg, e);
callback.onFailure(e);
}
});
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue()));
}
nfConsumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to obtain notifications from queue.", e);
try {
Thread.sleep(getNotificationPollDuration());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new notifications", e2);
}
}
}
protected abstract int getMgmtThreadPoolSize();
protected abstract TbQueueConsumer<TbProtoQueueMsg<N>> createNotificationsConsumer();
protected void processNotifications(List<TbProtoQueueMsg<N>> msgs, TbQueueConsumer<TbProtoQueueMsg<N>> consumer) throws Exception {
List<IdMsgPair<N>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).collect(Collectors.toList());
ConcurrentMap<UUID, TbProtoQueueMsg<N>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<N>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
orderedMsgList.forEach(element -> {
UUID id = element.getUuid();
TbProtoQueueMsg<N> msg = element.getMsg();
log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
handleNotification(id, msg, callback);
} catch (Throwable e) {
log.warn("[{}] Failed to process notification: {}", id, msg, e);
callback.onFailure(e);
}
log.info("TB Notifications Consumer stopped.");
});
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue()));
}
consumer.commit();
}
protected final void handleComponentLifecycleMsg(UUID id, ComponentLifecycleMsg componentLifecycleMsg) {
@ -201,13 +200,15 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
@PreDestroy
public void destroy() {
stopped = true;
stopConsumers();
if (nfConsumer != null) {
nfConsumer.unsubscribe();
if (consumersExecutor != null) {
consumersExecutor.shutdownNow();
}
if (mgmtExecutor != null) {
mgmtExecutor.shutdownNow();
}
if (notificationsConsumerExecutor != null) {
notificationsConsumerExecutor.shutdownNow();
if (scheduler != null) {
scheduler.shutdownNow();
}
}
}

8
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java

@ -18,7 +18,7 @@ package org.thingsboard.server.service.queue.ruleengine;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.ToString;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.queue.QueueConfig;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import java.util.Set;
@ -29,7 +29,7 @@ import java.util.Set;
public class TbQueueConsumerManagerTask {
private final QueueEvent event;
private Queue queue;
private QueueConfig config;
private Set<TopicPartitionInfo> partitions;
private boolean drainQueue;
@ -37,8 +37,8 @@ public class TbQueueConsumerManagerTask {
return new TbQueueConsumerManagerTask(QueueEvent.DELETE, null, null, drainQueue);
}
public static TbQueueConsumerManagerTask configUpdate(Queue queue) {
return new TbQueueConsumerManagerTask(QueueEvent.CONFIG_UPDATE, queue, null, false);
public static TbQueueConsumerManagerTask configUpdate(QueueConfig config) {
return new TbQueueConsumerManagerTask(QueueEvent.CONFIG_UPDATE, config, null, false);
}
public static TbQueueConsumerManagerTask partitionChange(Set<TopicPartitionInfo> partitions) {

7
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java

@ -20,9 +20,8 @@ import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.TbQueueMsg;
import java.util.Set;
import java.util.concurrent.Future;
@ -30,12 +29,12 @@ import java.util.concurrent.TimeUnit;
@RequiredArgsConstructor
@Slf4j
public class TbQueueConsumerTask {
public class TbQueueConsumerTask<M extends TbQueueMsg> {
@Getter
private final Object key;
@Getter
private final TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> consumer;
private final TbQueueConsumer<M> consumer;
@Setter
private Future<?> task;

25
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java

@ -19,8 +19,6 @@ import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.queue.TbQueueAdmin;
@ -33,11 +31,6 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStr
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory;
import org.thingsboard.server.service.stats.RuleEngineStatisticsService;
import javax.annotation.PostConstruct;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@Component
@TbRuleEngineComponent
@Slf4j
@ -68,22 +61,4 @@ public class TbRuleEngineConsumerContext {
private final TbQueueProducerProvider producerProvider;
private final TbQueueAdmin queueAdmin;
private ExecutorService consumersExecutor;
private ExecutorService mgmtExecutor;
private ScheduledExecutorService scheduler;
private volatile boolean isReady = false;
@PostConstruct
void init() {
this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer"));
this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(mgmtThreadPoolSize, "tb-rule-engine-mgmt");
this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler"));
}
public void stop() {
scheduler.shutdownNow();
consumersExecutor.shutdownNow();
mgmtExecutor.shutdownNow();
}
}

290
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java

@ -16,9 +16,8 @@
package org.thingsboard.server.service.queue.ruleengine;
import com.google.protobuf.ProtocolStringList;
import lombok.Getter;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
@ -38,173 +37,54 @@ import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.service.queue.TbMsgPackCallback;
import org.thingsboard.server.service.queue.TbMsgPackProcessingContext;
import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats;
import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Future;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
@Slf4j
public class TbRuleEngineQueueConsumerManager {
public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<TbProtoQueueMsg<ToRuleEngineMsg>, Queue> {
public static final String SUCCESSFUL_STATUS = "successful";
public static final String FAILED_STATUS = "failed";
private final TbRuleEngineConsumerContext ctx;
private final QueueKey queueKey;
private final TbRuleEngineConsumerStats stats;
private final ReentrantLock lock = new ReentrantLock(); //NonfairSync
@Getter
private volatile Queue queue;
@Getter
private volatile Set<TopicPartitionInfo> partitions;
private volatile ConsumerWrapper consumerWrapper;
private volatile boolean stopped;
private final java.util.Queue<TbQueueConsumerManagerTask> tasks = new ConcurrentLinkedQueue<>();
public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx, QueueKey queueKey) {
@Builder(builderMethodName = "create") // not to conflict with super.builder()
public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx,
QueueKey queueKey,
ExecutorService consumerExecutor,
ScheduledExecutorService scheduler,
ExecutorService taskExecutor) {
super(queueKey, null, null, ctx.getQueueFactory()::createToRuleEngineMsgConsumer, consumerExecutor, scheduler, taskExecutor);
this.ctx = ctx;
this.queueKey = queueKey;
this.stats = new TbRuleEngineConsumerStats(queueKey, ctx.getStatsFactory());
}
public void init(Queue queue) {
this.queue = queue;
if (queue.isConsumerPerPartition()) {
this.consumerWrapper = new ConsumerPerPartitionWrapper();
} else {
this.consumerWrapper = new SingleConsumerWrapper();
}
log.debug("[{}] Initialized consumer for queue: {}", queueKey, queue);
}
public void update(Queue queue) {
addTask(TbQueueConsumerManagerTask.configUpdate(queue));
}
public void update(Set<TopicPartitionInfo> partitions) {
addTask(TbQueueConsumerManagerTask.partitionChange(partitions));
}
public void delete(boolean drainQueue) {
addTask(TbQueueConsumerManagerTask.delete(drainQueue));
}
private void addTask(TbQueueConsumerManagerTask todo) {
if (stopped) {
return;
}
tasks.add(todo);
log.trace("[{}] Added task: {}", queueKey, todo);
tryProcessTasks();
}
private void tryProcessTasks() {
if (!ctx.isReady()) {
log.debug("[{}] TbRuleEngineConsumerContext is not ready yet, will process tasks later", queueKey);
ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS);
return;
}
ctx.getMgmtExecutor().submit(() -> {
if (lock.tryLock()) {
try {
Queue newConfiguration = null;
Set<TopicPartitionInfo> newPartitions = null;
while (!stopped) {
TbQueueConsumerManagerTask task = tasks.poll();
if (task == null) {
break;
}
log.trace("[{}] Processing task: {}", queueKey, task);
if (task.getEvent() == QueueEvent.PARTITION_CHANGE) {
newPartitions = task.getPartitions();
} else if (task.getEvent() == QueueEvent.CONFIG_UPDATE) {
newConfiguration = task.getQueue();
} else if (task.getEvent() == QueueEvent.DELETE) {
doDelete(task.isDrainQueue());
return;
}
}
if (stopped) {
return;
}
if (newConfiguration != null) {
doUpdate(newConfiguration);
}
if (newPartitions != null) {
doUpdate(newPartitions);
}
} catch (Exception e) {
log.error("[{}] Failed to process tasks", queueKey, e);
} finally {
lock.unlock();
}
} else {
log.trace("[{}] Failed to acquire lock", queueKey);
ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS);
}
});
}
private void doUpdate(Queue newQueue) {
log.info("[{}] Processing queue update: {}", queueKey, newQueue);
var oldQueue = this.queue;
this.queue = newQueue;
if (log.isTraceEnabled()) {
log.trace("[{}] Old queue configuration: {}", queueKey, oldQueue);
log.trace("[{}] New queue configuration: {}", queueKey, newQueue);
}
if (oldQueue == null) {
init(queue);
} else if (newQueue.isConsumerPerPartition() != oldQueue.isConsumerPerPartition()) {
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop);
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion);
init(queue);
if (partitions != null) {
doUpdate(partitions); // even if partitions number was changed, there can be no partition change event
}
} else {
// do nothing, because partitions change (if they changed) will be handled on PartitionChangeEvent,
// and changes to pollInterval/packProcessingTimeout/submitStrategy/processingStrategy will be picked up by consumer on the fly,
// and queue topic and name are immutable
@Override
protected void processTask(TbQueueConsumerManagerTask task) {
if (task.getEvent() == QueueEvent.DELETE) {
doDelete(task.isDrainQueue());
}
}
private void doUpdate(Set<TopicPartitionInfo> partitions) {
this.partitions = partitions;
consumerWrapper.updatePartitions(partitions);
}
public void stop() {
log.debug("[{}] Stopping consumers", queueKey);
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop);
stopped = true;
}
public void awaitStop() {
consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion);
log.debug("[{}] Unsubscribed and stopped consumers", queueKey);
}
private void doDelete(boolean drainQueue) {
stopped = true;
log.info("[{}] Handling queue deletion", queueKey);
@ -212,7 +92,7 @@ public class TbRuleEngineQueueConsumerManager {
List<TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>>> queueConsumers = consumerWrapper.getConsumers().stream()
.map(TbQueueConsumerTask::getConsumer).collect(Collectors.toList());
ctx.getConsumersExecutor().submit(() -> {
consumerExecutor.submit(() -> {
if (drainQueue) {
drainQueue(queueConsumers);
}
@ -235,47 +115,10 @@ public class TbRuleEngineQueueConsumerManager {
});
}
private void launchConsumer(TbQueueConsumerTask consumerTask) {
log.info("[{}] Launching consumer", consumerTask.getKey());
Future<?> consumerLoop = ctx.getConsumersExecutor().submit(() -> {
ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString());
try {
consumerLoop(consumerTask.getConsumer());
} catch (Throwable e) {
log.error("Failure in consumer loop", e);
}
});
consumerTask.setTask(consumerLoop);
}
private void consumerLoop(TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer) {
while (!stopped && !consumer.isStopped()) {
try {
List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs = consumer.poll(queue.getPollInterval());
if (msgs.isEmpty()) {
continue;
}
processMsgs(msgs, consumer, queue);
} catch (Exception e) {
if (!consumer.isStopped()) {
log.warn("Failed to process messages from queue", e);
try {
Thread.sleep(ctx.getPollDuration());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new requests", e2);
}
}
}
}
if (consumer.isStopped()) {
consumer.unsubscribe();
}
log.info("Rule Engine consumer stopped");
}
private void processMsgs(List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs,
TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer,
Queue queue) throws InterruptedException {
@Override
protected void processMsgs(List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs,
TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer,
Queue queue) throws Exception {
TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(queue);
TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue);
submitStrategy.init(msgs);
@ -320,7 +163,7 @@ public class TbRuleEngineQueueConsumerManager {
}
private void submitMessage(TbMsgPackProcessingContext packCtx, UUID id, TbProtoQueueMsg<ToRuleEngineMsg> msg) {
log.trace("[{}] Creating callback for topic {} message: {}", id, queue.getName(), msg.getValue());
log.trace("[{}] Creating callback for topic {} message: {}", id, config.getName(), msg.getValue());
ToRuleEngineMsg toRuleEngineMsg = msg.getValue();
TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB()));
TbMsgCallback callback = ctx.isPrometheusStatsEnabled() ?
@ -328,7 +171,7 @@ public class TbRuleEngineQueueConsumerManager {
new TbMsgPackCallback(id, tenantId, packCtx);
try {
if (!toRuleEngineMsg.getTbMsg().isEmpty()) {
forwardToRuleEngineActor(queue.getName(), tenantId, toRuleEngineMsg, callback);
forwardToRuleEngineActor(config.getName(), tenantId, toRuleEngineMsg, callback);
} else {
callback.onSuccess();
}
@ -356,7 +199,7 @@ public class TbRuleEngineQueueConsumerManager {
log.info("[{}] {} to process [{}] messages", queueKey, prefix, map.size());
for (Map.Entry<UUID, TbProtoQueueMsg<ToRuleEngineMsg>> pending : map.entrySet()) {
ToRuleEngineMsg tmp = pending.getValue().getValue();
TbMsg tmpMsg = TbMsg.fromBytes(queue.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY);
TbMsg tmpMsg = TbMsg.fromBytes(config.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY);
RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey());
if (printAll) {
log.trace("[{}][{}] {} to process message: {}, Last Rule Node: {}", queueKey, TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo);
@ -379,7 +222,7 @@ public class TbRuleEngineQueueConsumerManager {
int n = 0;
while (System.currentTimeMillis() <= finishTs) {
for (TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer : consumers) {
List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs = consumer.poll(queue.getPollInterval());
List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs = consumer.poll(config.getPollInterval());
if (msgs.isEmpty()) {
continue;
}
@ -388,7 +231,7 @@ public class TbRuleEngineQueueConsumerManager {
MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray());
EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB()));
TopicPartitionInfo tpi = ctx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator);
TopicPartitionInfo tpi = ctx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, config.getName(), TenantId.SYS_TENANT_ID, originator);
ctx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, msg, null);
n++;
} catch (Throwable e) {
@ -399,90 +242,11 @@ public class TbRuleEngineQueueConsumerManager {
}
}
if (n > 0) {
log.info("Moved {} messages from {} to system {}", n, queueKey, queue.getName());
log.info("Moved {} messages from {} to system {}", n, queueKey, config.getName());
}
} catch (Exception e) {
log.error("[{}] Failed to drain queue", queueKey, e);
}
}
private static String partitionsToString(Collection<TopicPartitionInfo> partitions) {
return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.joining(", ", "[", "]"));
}
interface ConsumerWrapper {
void updatePartitions(Set<TopicPartitionInfo> partitions);
Collection<TbQueueConsumerTask> getConsumers();
}
class ConsumerPerPartitionWrapper implements ConsumerWrapper {
private final Map<TopicPartitionInfo, TbQueueConsumerTask> consumers = new HashMap<>();
@Override
public void updatePartitions(Set<TopicPartitionInfo> partitions) {
Set<TopicPartitionInfo> addedPartitions = new HashSet<>(partitions);
addedPartitions.removeAll(consumers.keySet());
Set<TopicPartitionInfo> removedPartitions = new HashSet<>(consumers.keySet());
removedPartitions.removeAll(partitions);
log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, partitionsToString(addedPartitions), partitionsToString(removedPartitions));
removedPartitions.forEach((tpi) -> {
consumers.get(tpi).initiateStop();
});
removedPartitions.forEach((tpi) -> {
consumers.remove(tpi).awaitCompletion();
});
addedPartitions.forEach((tpi) -> {
String key = queueKey + "-" + tpi.getPartition().orElse(-999999);
TbQueueConsumerTask consumer = new TbQueueConsumerTask(key, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue));
consumers.put(tpi, consumer);
consumer.subscribe(Set.of(tpi));
launchConsumer(consumer);
});
}
@Override
public Collection<TbQueueConsumerTask> getConsumers() {
return consumers.values();
}
}
class SingleConsumerWrapper implements ConsumerWrapper {
private TbQueueConsumerTask consumer;
@Override
public void updatePartitions(Set<TopicPartitionInfo> partitions) {
log.info("[{}] New partitions: {}", queueKey, partitionsToString(partitions));
if (partitions.isEmpty()) {
if (consumer != null && consumer.isRunning()) {
consumer.initiateStop();
consumer.awaitCompletion();
}
consumer = null;
return;
}
if (consumer == null) {
consumer = new TbQueueConsumerTask(queueKey, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue));
}
consumer.subscribe(partitions);
if (!consumer.isRunning()) {
launchConsumer(consumer);
}
}
@Override
public Collection<TbQueueConsumerTask> getConsumers() {
if (consumer == null) {
return Collections.emptyList();
}
return List.of(consumer);
}
}
}

80
application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsService.java

@ -17,25 +17,24 @@ package org.thingsboard.server.service.security.auth.jwt.settings;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.notification.targets.platform.SystemAdministratorsFilter;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.dao.notification.DefaultNotifications;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Objects;
import java.util.Optional;
import static org.thingsboard.server.service.security.model.token.JwtTokenFactory.KEY_LENGTH;
@Service
@RequiredArgsConstructor
@Slf4j
@ -43,49 +42,11 @@ public class DefaultJwtSettingsService implements JwtSettingsService {
private final AdminSettingsService adminSettingsService;
private final Optional<TbClusterService> tbClusterService;
private final Optional<NotificationCenter> notificationCenter;
private final JwtSettingsValidator jwtSettingsValidator;
@Value("${security.jwt.tokenExpirationTime:9000}")
private Integer tokenExpirationTime;
@Value("${security.jwt.refreshTokenExpTime:604800}")
private Integer refreshTokenExpTime;
@Value("${security.jwt.tokenIssuer:thingsboard.io}")
private String tokenIssuer;
@Value("${security.jwt.tokenSigningKey:thingsboardDefaultSigningKey}")
private String tokenSigningKey;
private final Optional<JwtTokenFactory> jwtTokenFactory;
private volatile JwtSettings jwtSettings = null; //lazy init
/**
* Create JWT admin settings is intended to be called from Install scripts only
*/
@Override
public void createRandomJwtSettings() {
if (getJwtSettingsFromDb() == null) {
log.info("Creating JWT admin settings...");
this.jwtSettings = getJwtSettingsFromYml();
if (isSigningKeyDefault(jwtSettings)) {
this.jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(
RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8)));
}
saveJwtSettings(jwtSettings);
} else {
log.info("Skip creating JWT admin settings because they already exist.");
}
}
/**
* Create JWT admin settings is intended to be called from Upgrade scripts only
*/
@Override
public void saveLegacyYmlSettings() {
log.info("Saving legacy JWT admin settings from YML...");
if (getJwtSettingsFromDb() == null) {
saveJwtSettings(getJwtSettingsFromYml());
}
}
@Override
public JwtSettings saveJwtSettings(JwtSettings jwtSettings) {
jwtSettingsValidator.validate(jwtSettings);
@ -105,7 +66,9 @@ public class DefaultJwtSettingsService implements JwtSettingsService {
@Override
public JwtSettings reloadJwtSettings() {
log.trace("Executing reloadJwtSettings");
return getJwtSettings(true);
var settings = getJwtSettings(true);
jwtTokenFactory.ifPresent(JwtTokenFactory::reload);
return settings;
}
@Override
@ -118,30 +81,13 @@ public class DefaultJwtSettingsService implements JwtSettingsService {
if (this.jwtSettings == null || forceReload) {
synchronized (this) {
if (this.jwtSettings == null || forceReload) {
JwtSettings result = getJwtSettingsFromDb();
if (result == null) {
result = getJwtSettingsFromYml();
log.warn("Loading the JWT settings from YML since there are no settings in DB. Looks like the upgrade script was not applied.");
}
if (isSigningKeyDefault(result)) {
log.warn("WARNING: The platform is configured to use default JWT Signing Key. " +
"This is a security issue that needs to be resolved. Please change the JWT Signing Key using the Web UI. " +
"Navigate to \"System settings -> Security settings\" while logged in as a System Administrator.");
notificationCenter.ifPresent(notificationCenter -> {
notificationCenter.sendGeneralWebNotification(TenantId.SYS_TENANT_ID, new SystemAdministratorsFilter(), DefaultNotifications.jwtSigningKeyIssue.toTemplate());
});
}
this.jwtSettings = result;
jwtSettings = getJwtSettingsFromDb();
}
}
}
return this.jwtSettings;
}
private JwtSettings getJwtSettingsFromYml() {
return new JwtSettings(this.tokenExpirationTime, this.refreshTokenExpTime, this.tokenIssuer, this.tokenSigningKey);
}
private JwtSettings getJwtSettingsFromDb() {
AdminSettings adminJwtSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY);
return adminJwtSettings != null ? mapAdminToJwtSettings(adminJwtSettings) : null;
@ -161,8 +107,12 @@ public class DefaultJwtSettingsService implements JwtSettingsService {
return adminJwtSettings;
}
private boolean isSigningKeyDefault(JwtSettings settings) {
public static boolean isSigningKeyDefault(JwtSettings settings) {
return TOKEN_SIGNING_KEY_DEFAULT.equals(settings.getTokenSigningKey());
}
public static boolean validateTokenSigningKeyLength(JwtSettings settings) {
return Base64.getDecoder().decode(settings.getTokenSigningKey()).length * Byte.SIZE >= KEY_LENGTH;
}
}

7
application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java

@ -27,6 +27,9 @@ import java.util.Base64;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import static org.thingsboard.server.service.security.auth.jwt.settings.DefaultJwtSettingsService.isSigningKeyDefault;
import static org.thingsboard.server.service.security.model.token.JwtTokenFactory.KEY_LENGTH;
@Component
@RequiredArgsConstructor
public class DefaultJwtSettingsValidator implements JwtSettingsValidator {
@ -59,8 +62,8 @@ public class DefaultJwtSettingsValidator implements JwtSettingsValidator {
if (Arrays.isNullOrEmpty(decodedKey)) {
throw new DataValidationException("JWT token signing key should be non-empty after Base64 decoding!");
}
if (decodedKey.length * Byte.SIZE < 256 && !JwtSettingsService.TOKEN_SIGNING_KEY_DEFAULT.equals(jwtSettings.getTokenSigningKey())) {
throw new DataValidationException("JWT token signing key should be a Base64 encoded string representing at least 256 bits of data!");
if (decodedKey.length * Byte.SIZE < KEY_LENGTH && !isSigningKeyDefault(jwtSettings)) {
throw new DataValidationException("JWT token signing key should be a Base64 encoded string representing at least 512 bits of data!");
}
System.arraycopy(decodedKey, 0, RandomUtils.nextBytes(decodedKey.length), 0, decodedKey.length); //secure memory

4
application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsService.java

@ -26,10 +26,6 @@ public interface JwtSettingsService {
JwtSettings reloadJwtSettings();
void createRandomJwtSettings();
void saveLegacyYmlSettings();
JwtSettings saveJwtSettings(JwtSettings jwtSettings);
}

72
application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java

@ -16,16 +16,19 @@
package org.thingsboard.server.service.security.model.token;
import io.jsonwebtoken.Claims;
import io.jsonwebtoken.ClaimsBuilder;
import io.jsonwebtoken.ExpiredJwtException;
import io.jsonwebtoken.Jws;
import io.jsonwebtoken.JwtBuilder;
import io.jsonwebtoken.JwtParser;
import io.jsonwebtoken.Jwts;
import io.jsonwebtoken.MalformedJwtException;
import io.jsonwebtoken.SignatureAlgorithm;
import io.jsonwebtoken.SignatureException;
import io.jsonwebtoken.UnsupportedJwtException;
import io.jsonwebtoken.security.Keys;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Lazy;
import org.springframework.security.authentication.BadCredentialsException;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.stereotype.Component;
@ -41,7 +44,10 @@ import org.thingsboard.server.service.security.exception.JwtExpiredTokenExceptio
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.time.ZonedDateTime;
import java.util.Base64;
import java.util.Collections;
import java.util.Date;
import java.util.List;
@ -53,6 +59,8 @@ import java.util.stream.Collectors;
@Slf4j
public class JwtTokenFactory {
public static int KEY_LENGTH = Jwts.SIG.HS512.getKeyBitLength();
private static final String SCOPES = "scopes";
private static final String USER_ID = "userId";
private static final String FIRST_NAME = "firstName";
@ -63,8 +71,12 @@ public class JwtTokenFactory {
private static final String CUSTOMER_ID = "customerId";
private static final String SESSION_ID = "sessionId";
@Lazy
private final JwtSettingsService jwtSettingsService;
private volatile JwtParser jwtParser;
private volatile SecretKey secretKey;
/**
* Factory method for issuing new JWT Tokens.
*/
@ -95,7 +107,7 @@ public class JwtTokenFactory {
public SecurityUser parseAccessJwtToken(String token) {
Jws<Claims> jwsClaims = parseTokenClaims(token);
Claims claims = jwsClaims.getBody();
Claims claims = jwsClaims.getPayload();
String subject = claims.getSubject();
@SuppressWarnings("unchecked")
List<String> scopes = claims.get(SCOPES, List.class);
@ -140,14 +152,14 @@ public class JwtTokenFactory {
String token = setUpToken(securityUser, Collections.singletonList(Authority.REFRESH_TOKEN.name()), jwtSettingsService.getJwtSettings().getRefreshTokenExpTime())
.claim(IS_PUBLIC, principal.getType() == UserPrincipal.Type.PUBLIC_ID)
.setId(UUID.randomUUID().toString()).compact();
.id(UUID.randomUUID().toString()).compact();
return new AccessJwtToken(token);
}
public SecurityUser parseRefreshToken(String token) {
Jws<Claims> jwsClaims = parseTokenClaims(token);
Claims claims = jwsClaims.getBody();
Claims claims = jwsClaims.getPayload();
String subject = claims.getSubject();
@SuppressWarnings("unchecked")
List<String> scopes = claims.get(SCOPES, List.class);
@ -176,6 +188,11 @@ public class JwtTokenFactory {
return new AccessJwtToken(jwtBuilder.compact());
}
public void reload() {
getSecretKey(true);
getJwtParser(true);
}
private JwtBuilder setUpToken(SecurityUser securityUser, List<String> scopes, long expirationTime) {
if (StringUtils.isBlank(securityUser.getEmail())) {
throw new IllegalArgumentException("Cannot create JWT Token without username/email");
@ -183,28 +200,27 @@ public class JwtTokenFactory {
UserPrincipal principal = securityUser.getUserPrincipal();
Claims claims = Jwts.claims().setSubject(principal.getValue());
claims.put(USER_ID, securityUser.getId().getId().toString());
claims.put(SCOPES, scopes);
ClaimsBuilder claimsBuilder = Jwts.claims()
.subject(principal.getValue())
.add(USER_ID, securityUser.getId().getId().toString())
.add(SCOPES, scopes);
if (securityUser.getSessionId() != null) {
claims.put(SESSION_ID, securityUser.getSessionId());
claimsBuilder.add(SESSION_ID, securityUser.getSessionId());
}
ZonedDateTime currentTime = ZonedDateTime.now();
return Jwts.builder()
.setClaims(claims)
.setIssuer(jwtSettingsService.getJwtSettings().getTokenIssuer())
.setIssuedAt(Date.from(currentTime.toInstant()))
.setExpiration(Date.from(currentTime.plusSeconds(expirationTime).toInstant()))
.signWith(SignatureAlgorithm.HS512, jwtSettingsService.getJwtSettings().getTokenSigningKey());
.claims(claimsBuilder.build())
.issuer(jwtSettingsService.getJwtSettings().getTokenIssuer())
.issuedAt(Date.from(currentTime.toInstant()))
.expiration(Date.from(currentTime.plusSeconds(expirationTime).toInstant()))
.signWith(getSecretKey(false), Jwts.SIG.HS512);
}
public Jws<Claims> parseTokenClaims(String token) {
try {
return Jwts.parser()
.setSigningKey(jwtSettingsService.getJwtSettings().getTokenSigningKey())
.parseClaimsJws(token);
return getJwtParser(false).parseSignedClaims(token);
} catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException ex) {
log.debug("Invalid JWT Token", ex);
throw new BadCredentialsException("Invalid JWT token: ", ex);
@ -220,4 +236,28 @@ public class JwtTokenFactory {
return new JwtPair(accessToken.getToken(), refreshToken.getToken());
}
private SecretKey getSecretKey(boolean forceReload) {
if (secretKey == null || forceReload) {
synchronized (this) {
if (secretKey == null || forceReload) {
byte[] decodedToken = Base64.getDecoder().decode(jwtSettingsService.getJwtSettings().getTokenSigningKey());
secretKey = new SecretKeySpec(decodedToken, "HmacSHA512");
}
}
}
return secretKey;
}
private JwtParser getJwtParser(boolean forceReload) {
if (jwtParser == null || forceReload) {
synchronized (this) {
if (jwtParser == null || forceReload) {
jwtParser = Jwts.parser()
.verifyWith(Keys.hmacShaKeyFor(Base64.getDecoder().decode(jwtSettingsService.getJwtSettings().getTokenSigningKey())))
.build();
}
}
}
return jwtParser;
}
}

6
application/src/main/java/org/thingsboard/server/service/security/model/token/OAuth2AppTokenFactory.java

@ -22,10 +22,12 @@ import io.jsonwebtoken.Jwts;
import io.jsonwebtoken.MalformedJwtException;
import io.jsonwebtoken.SignatureException;
import io.jsonwebtoken.UnsupportedJwtException;
import io.jsonwebtoken.security.Keys;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.StringUtils;
import java.util.Base64;
import java.util.Date;
import java.util.concurrent.TimeUnit;
@ -40,14 +42,14 @@ public class OAuth2AppTokenFactory {
public String validateTokenAndGetCallbackUrlScheme(String appPackage, String appToken, String appSecret) {
Jws<Claims> jwsClaims;
try {
jwsClaims = Jwts.parser().setSigningKey(appSecret).parseClaimsJws(appToken);
jwsClaims = Jwts.parser().verifyWith(Keys.hmacShaKeyFor(Base64.getDecoder().decode(appSecret))).build().parseSignedClaims(appToken);
}
catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException | SignatureException ex) {
throw new IllegalArgumentException("Invalid Application token: ", ex);
} catch (ExpiredJwtException expiredEx) {
throw new IllegalArgumentException("Application token expired", expiredEx);
}
Claims claims = jwsClaims.getBody();
Claims claims = jwsClaims.getPayload();
Date expiration = claims.getExpiration();
if (expiration == null) {
throw new IllegalArgumentException("Application token must have expiration date");

28
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java

@ -108,34 +108,40 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi
return entity;
}
@Override
public <E extends ExportableEntity<I>, I extends EntityId> E findDefaultEntityByTenantId(TenantId tenantId, EntityType entityType) {
ExportableEntityDao<I, E> dao = getExportableEntityDao(entityType);
if (dao == null) {
return null;
}
return dao.findDefaultEntityByTenantId(tenantId.getId());
}
@Override
public <E extends ExportableEntity<I>, I extends EntityId> PageData<E> findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) {
ExportableEntityDao<I, E> dao = getExportableEntityDao(entityType);
if (dao != null) {
return dao.findByTenantId(tenantId.getId(), pageLink);
} else {
return new PageData<>();
if (dao == null) {
return PageData.emptyPageData();
}
return dao.findByTenantId(tenantId.getId(), pageLink);
}
@Override
public <I extends EntityId> PageData<I> findEntitiesIdsByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) {
ExportableEntityDao<I, ?> dao = getExportableEntityDao(entityType);
if (dao != null) {
return dao.findIdsByTenantId(tenantId.getId(), pageLink);
} else {
return new PageData<>();
if (dao == null) {
return PageData.emptyPageData();
}
return dao.findIdsByTenantId(tenantId.getId(), pageLink);
}
@Override
public <I extends EntityId> I getExternalIdByInternal(I internalId) {
ExportableEntityDao<I, ?> dao = getExportableEntityDao(internalId.getEntityType());
if (dao != null) {
return dao.getExternalIdByInternal(internalId);
} else {
if (dao == null) {
return null;
}
return dao.getExternalIdByInternal(internalId);
}
private boolean belongsToTenant(HasId<? extends EntityId> entity, TenantId tenantId) {

2
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java

@ -33,6 +33,8 @@ public interface ExportableEntitiesService {
<E extends ExportableEntity<I>, I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name);
<E extends ExportableEntity<I>, I extends EntityId> E findDefaultEntityByTenantId(TenantId tenantId, EntityType entityType);
<E extends ExportableEntity<I>, I extends EntityId> PageData<E> findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink);
<I extends EntityId> PageData<I> findEntitiesIdsByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink);

21
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java

@ -27,6 +27,7 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasDefaultOption;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
@ -73,7 +74,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
@Autowired
@Lazy
private ExportableEntitiesService exportableEntitiesService;
private ExportableEntitiesService entitiesService;
@Autowired
private RelationService relationService;
@Autowired
@ -274,22 +275,30 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
@SuppressWarnings("unchecked")
protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) {
return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId()))
.or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId())))
return (E) Optional.ofNullable(entitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId()))
.or(() -> Optional.ofNullable(entitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId())))
.or(() -> {
if (ctx.isFindExistingByName()) {
return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(ctx.getTenantId(), getEntityType(), entity.getName()));
return Optional.ofNullable(entitiesService.findEntityByTenantIdAndName(ctx.getTenantId(), getEntityType(), entity.getName()));
} else {
return Optional.empty();
}
})
.or(() -> {
if (entity instanceof HasDefaultOption hasDefaultOption) {
if (hasDefaultOption.isDefault()) {
return Optional.ofNullable(entitiesService.findDefaultEntityByTenantId(ctx.getTenantId(), getEntityType()));
}
}
return Optional.empty();
})
.orElse(null);
}
@SuppressWarnings("unchecked")
private <ID extends EntityId> HasId<ID> findInternalEntity(TenantId tenantId, ID externalId) {
return (HasId<ID>) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId))
.or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId)))
return (HasId<ID>) Optional.ofNullable(entitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId))
.or(() -> Optional.ofNullable(entitiesService.findEntityByTenantIdAndId(tenantId, externalId)))
.orElseThrow(() -> new MissingEntityException(externalId));
}

10
application/src/main/resources/thingsboard.yml

@ -164,6 +164,8 @@ mail:
oauth2:
# Interval for checking refresh token expiration in seconds(by default, 1 day).
refreshTokenCheckingInterval: "${REFRESH_TOKEN_EXPIRATION_CHECKING_INTERVAL:86400}"
# Rate limits for sending mails per tenant. As example for 1000 per minute and 10000 per hour is "1000:60,10000:3600"
per_tenant_rate_limits: "${MAIL_PER_TENANT_RATE_LIMITS:}"
# Usage statistics parameters
usage:
@ -505,6 +507,12 @@ cache:
assets:
timeToLiveInMinutes: "${CACHE_SPECS_ASSETS_TTL:1440}" # Asset cache TTL
maxSize: "${CACHE_SPECS_ASSETS_MAX_SIZE:10000}" # 0 means the cache is disabled
customers:
timeToLiveInMinutes: "${CACHE_SPECS_CUSTOMERS_TTL:1440}" # Customer cache TTL
maxSize: "${CACHE_SPECS_CUSTOMERS_MAX_SIZE:10000}" # 0 means the cache is disabled
users:
timeToLiveInMinutes: "${CACHE_SPECS_USERS_TTL:1440}" # User cache TTL
maxSize: "${CACHE_SPECS_USERS_MAX_SIZE:10000}" # 0 means the cache is disabled
entityViews:
timeToLiveInMinutes: "${CACHE_SPECS_ENTITY_VIEWS_TTL:1440}" # Entity view cache TTL
maxSize: "${CACHE_SPECS_ENTITY_VIEWS_MAX_SIZE:10000}" # 0 means the cache is disabled
@ -1621,6 +1629,8 @@ queue:
partitions: "${TB_QUEUE_CORE_PARTITIONS:10}"
# Timeout for processing a message pack by Core microservices
pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}"
# Enable/disable a separate consumer per partition for Core queue
consumer-per-partition: "${TB_QUEUE_CORE_CONSUMER_PER_PARTITION:true}"
ota:
# Default topic name for OTA updates
topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}"

17
application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@ -21,9 +21,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import io.jsonwebtoken.Claims;
import io.jsonwebtoken.Header;
import io.jsonwebtoken.Jwt;
import io.jsonwebtoken.Jwts;
import io.jsonwebtoken.Jws;
import lombok.extern.slf4j.Slf4j;
import org.awaitility.Awaitility;
import org.hamcrest.Matcher;
@ -121,6 +119,7 @@ import org.thingsboard.server.queue.memory.InMemoryStorage;
import org.thingsboard.server.service.entitiy.tenant.profile.TbTenantProfileService;
import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRequest;
import org.thingsboard.server.service.security.auth.rest.LoginRequest;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
@ -241,6 +240,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
@Autowired
protected ClaimDevicesService claimDevicesService;
@Autowired
private JwtTokenFactory jwtTokenFactory;
@SpyBean
protected MailService mailService;
@ -561,13 +563,8 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
}
protected void validateJwtToken(String token, String username) {
Assert.assertNotNull(token);
Assert.assertFalse(token.isEmpty());
int i = token.lastIndexOf('.');
Assert.assertTrue(i > 0);
String withoutSignature = token.substring(0, i + 1);
Jwt<Header, Claims> jwsClaims = Jwts.parser().parseClaimsJwt(withoutSignature);
Claims claims = jwsClaims.getBody();
Jws<Claims> jwsClaims = jwtTokenFactory.parseTokenClaims(token);
Claims claims = jwsClaims.getPayload();
String subject = claims.getSubject();
Assert.assertEquals(username, subject);
}

4
application/src/test/java/org/thingsboard/server/controller/AdminControllerTest.java

@ -42,7 +42,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
@Slf4j
@DaoSqlTest
public class AdminControllerTest extends AbstractControllerTest {
final JwtSettings defaultJwtSettings = new JwtSettings(9000, 604800, "thingsboard.io", "thingsboardDefaultSigningKey");
final JwtSettings defaultJwtSettings = new JwtSettings(9000, 604800, "thingsboard.io", "QmlicmJkZk9tSzZPVFozcWY0Sm94UVhybmtBWXZ5YmZMOUZSZzZvcUFiOVhsb3VHUThhUWJGaXp3UHhtcGZ6Tw==");
@Test
public void testFindAdminSettingsByKey() throws Exception {
@ -168,7 +168,7 @@ public class AdminControllerTest extends AbstractControllerTest {
assertThat(jwtSettings).isEqualTo(defaultJwtSettings);
jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(
RandomStringUtils.randomAlphanumeric(256 / Byte.SIZE).getBytes(StandardCharsets.UTF_8)));
RandomStringUtils.randomAlphanumeric(512 / Byte.SIZE).getBytes(StandardCharsets.UTF_8)));
doPost("/api/admin/jwtSettings", jwtSettings).andExpect(status().isOk());

7
application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java

@ -216,11 +216,12 @@ public class AuditLogControllerTest extends AbstractControllerTest {
@Test
public void whenSavingAuditLogAndPartitionSaveErrorOccurred_thenSaveAuditLogAnyway() throws Exception {
// creating partition bigger than sql.audit_logs.partition_size
long entityTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-04-29T07:43:11Z").getTime();
long entityTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-29T07:43:11Z").getTime();
//the partition 7 days is overlapping default partition size 1 day, use in the far past to not affect other tests
partitioningRepository.createPartitionIfNotExists("audit_log", entityTs, TimeUnit.DAYS.toMillis(7));
List<Long> partitions = partitioningRepository.fetchPartitions("audit_log");
log.warn("entityTs [{}], fetched partitions {}", entityTs, partitions);
assertThat(partitions).contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-04-29T00:00:00Z").getTime());
assertThat(partitions).contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-28T00:00:00Z").getTime());
partitioningRepository.cleanupPartitionsCache("audit_log", entityTs, 0);
assertDoesNotThrow(() -> {
@ -228,7 +229,7 @@ public class AuditLogControllerTest extends AbstractControllerTest {
createAuditLog(ActionType.LOGIN, tenantAdminUserId, entityTs);
});
assertThat(partitioningRepository.fetchPartitions("audit_log"))
.contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-04-29T00:00:00Z").getTime());;
.contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-28T00:00:00Z").getTime());;
}
private AuditLog createAuditLog(ActionType actionType, EntityId entityId, long entityTs) {

6
application/src/test/java/org/thingsboard/server/controller/AuthControllerTest.java

@ -167,4 +167,10 @@ public class AuthControllerTest extends AbstractControllerTest {
.andExpect(jsonPath("$.message",
is("Password must be no more than 72 characters in length.")));
}
@Test
public void testGetPageWithoutRedirect() throws Exception {
doGet("/login").andExpect(status().isOk());
doGet("/home").andExpect(status().isOk());
}
}

27
application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java

@ -352,7 +352,7 @@ public class CustomerControllerTest extends AbstractControllerTest {
}
@Test
public void testFindCustomersByTitle() throws Exception {
public void testFindCustomersWithTitleAsTextSearch() throws Exception {
TenantId tenantId = savedTenant.getId();
String title1 = "Customer title 1";
@ -424,6 +424,31 @@ public class CustomerControllerTest extends AbstractControllerTest {
Assert.assertEquals(0, pageData.getData().size());
}
@Test
public void testFindCustomerByTitle() throws Exception {
Customer customer = new Customer();
customer.setTitle("My customer");
Mockito.reset(tbClusterService, auditLogService);
Customer savedCustomer = doPost("/api/customer", customer, Customer.class);
testNotifyEntityAllOneTime(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), savedCustomer.getTenantId(),
new CustomerId(CustomerId.NULL_UUID), tenantAdmin.getId(), tenantAdmin.getEmail(),
ActionType.ADDED);
Assert.assertNotNull(savedCustomer);
Assert.assertNotNull(savedCustomer.getId());
Assert.assertTrue(savedCustomer.getCreatedTime() > 0);
Assert.assertEquals(customer.getTitle(), savedCustomer.getTitle());
Customer foundCustomer = doGet("/api/tenant/customers?customerTitle=" + savedCustomer.getTitle(), Customer.class);
Assert.assertEquals(foundCustomer, savedCustomer);
doDelete("/api/customer/" + savedCustomer.getId().getId().toString())
.andExpect(status().isOk());
}
@Test
public void testDeleteCustomerWithDeleteRelationsOk() throws Exception {
CustomerId customerId = createCustomer("Customer for Test WithRelationsOk").getId();

38
application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java

@ -84,6 +84,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.hamcrest.Matchers.containsString;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
@ -1541,17 +1542,24 @@ public class DeviceControllerTest extends AbstractControllerTest {
Device savedDevice = doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class);
//check server attribute value
List<Map<String, Object>> values = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<>() {
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
Map<String, Object> actualAttribute = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<List<Map<String, Object>>>() {}).stream()
.filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(attributeValue, actualAttribute.get("value"));
});
Map<String, Object> serverAttribute = values.stream().filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(attributeValue, serverAttribute.get("value"));
//update server attribute value
String newAttributeValue = "testValue2";
JsonNode content = JacksonUtil.toJsonNode("{\"DATA\": \"" + newAttributeValue + "\"}");
doPost("/api/plugins/telemetry/" + EntityType.DEVICE.name() + "/" + savedDevice.getUuidId() + "/SERVER_SCOPE", content)
.andExpect(status().isOk());
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
Map<String, Object> actualAttribute = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<List<Map<String, Object>>>() {}).stream()
.filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(newAttributeValue, actualAttribute.get("value"));
});
//reimport devices
String deviceName2 = "secondDevice";
@ -1563,19 +1571,19 @@ public class DeviceControllerTest extends AbstractControllerTest {
doPostWithTypedResponse("/api/device/bulk_import", request, new TypeReference<>() {});
Device savedDevice2 = doGet("/api/tenant/devices?deviceName=" + deviceName2, Device.class);
//check attribute value was not changed after reimport
List<Map<String, Object>> values2 = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<>() {
});
Map<String, Object> retrievedServerAttribute2 = values2.stream().filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(newAttributeValue, retrievedServerAttribute2.get("value"));
//check attribute for second device
List<Map<String, Object>> values3 = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice2.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<>() {
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
Map<String, Object> actualAttribute = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice2.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<List<Map<String, Object>>>() {}).stream()
.filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(attributeValue2, actualAttribute.get("value"));
});
Map<String, Object> retrievedServerAttribute3 = values3.stream().filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(attributeValue2, retrievedServerAttribute3.get("value"));
//check attribute value was not changed after reimport
Map<String, Object> actualAttribute = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<List<Map<String, Object>>>() {}).stream()
.filter(att -> att.get("key").equals("DATA")).findFirst().get();
Assert.assertEquals(newAttributeValue, actualAttribute.get("value"));
}
private Device createDevice(String name) {

13
application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java

@ -275,10 +275,9 @@ public class EntityViewControllerTest extends AbstractControllerTest {
}
@Test
public void testAssignAndUnAssignedEntityViewToCustomer() throws Exception {
public void testAssignAndUnassignEntityViewToCustomer() throws Exception {
EntityView view = getNewSavedEntityView("Test entity view");
Customer savedCustomer = doPost("/api/customer", getNewCustomer("My customer"), Customer.class);
view.setCustomerId(savedCustomer.getId());
Mockito.reset(tbClusterService, auditLogService);
@ -304,17 +303,17 @@ public class EntityViewControllerTest extends AbstractControllerTest {
ActionType.ASSIGNED_TO_CUSTOMER, ActionType.UPDATED,
foundView.getId().getId().toString(), foundView.getCustomerId().getId().toString(), savedCustomer.getTitle());
EntityView unAssignedView = doDelete("/api/customer/entityView/" + savedView.getId().getId().toString(), EntityView.class);
assertEquals(ModelConstants.NULL_UUID, unAssignedView.getCustomerId().getId());
EntityView unassignedView = doDelete("/api/customer/entityView/" + savedView.getId().getId().toString(), EntityView.class);
assertEquals(ModelConstants.NULL_UUID, unassignedView.getCustomerId().getId());
foundView = doGet("/api/entityView/" + savedView.getId().getId().toString(), EntityView.class);
assertEquals(ModelConstants.NULL_UUID, foundView.getCustomerId().getId());
testBroadcastEntityStateChangeEventTime(foundView.getId(), foundView.getTenantId(), 1);
testNotifyAssignUnassignEntityAllOneTime(unAssignedView, savedView.getId(), savedView.getId(),
tenantId, savedView.getCustomerId(), tenantAdminUserId, TENANT_ADMIN_EMAIL,
testNotifyAssignUnassignEntityAllOneTime(unassignedView, unassignedView.getId(), unassignedView.getId(),
tenantId, savedCustomer.getId(), tenantAdminUserId, TENANT_ADMIN_EMAIL,
ActionType.UNASSIGNED_FROM_CUSTOMER, ActionType.UPDATED,
assignedView.getId().getId().toString(), savedView.getCustomerId().getId().toString(), savedCustomer.getTitle());
assignedView.getId().getId().toString(), savedCustomer.getId().getId().toString(), savedCustomer.getTitle());
}
@Test

2
application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java

@ -279,7 +279,7 @@ public class UserControllerTest extends AbstractControllerTest {
user.setTenantId(tenantId);
user.setEmail(TENANT_ADMIN_EMAIL);
String msgError = "User with email '" + TENANT_ADMIN_EMAIL + "' already present in database";
String msgError = "User with email '" + TENANT_ADMIN_EMAIL + "' already present in database!";
doPost("/api/user", user)
.andExpect(status().isBadRequest())
.andExpect(statusReason(containsString(msgError)));

62
application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java

@ -27,6 +27,8 @@ import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.testcontainers.shaded.org.apache.commons.lang3.RandomUtils;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.QueueId;
@ -65,6 +67,9 @@ import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
@ -86,7 +91,6 @@ import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
@ -112,6 +116,9 @@ public class TbRuleEngineQueueConsumerManagerTest {
@Mock
private TbQueueAdmin queueAdmin;
private TbRuleEngineConsumerContext ruleEngineConsumerContext;
private ExecutorService consumersExecutor;
private ScheduledExecutorService scheduler;
private ExecutorService mgmtExecutor;
private TbRuleEngineQueueConsumerManager consumerManager;
private Queue queue;
@ -141,10 +148,10 @@ public class TbRuleEngineQueueConsumerManagerTest {
}).when(actorContext).tell(any());
ruleEngineMsgProducer = mock(TbQueueProducer.class);
when(producerProvider.getRuleEngineMsgProducer()).thenReturn(ruleEngineMsgProducer);
ruleEngineConsumerContext.setMgmtThreadPoolSize(2);
consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer"));
mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(3, "tb-rule-engine-mgmt");
scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler"));
ruleEngineConsumerContext.setTopicDeletionDelayInSec(5);
ruleEngineConsumerContext.init();
ruleEngineConsumerContext.setReady(false);
queue = new Queue();
queue.setName("Test");
@ -174,14 +181,23 @@ public class TbRuleEngineQueueConsumerManagerTest {
}).when(queueFactory).createToRuleEngineMsgConsumer(any());
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queue);
consumerManager = new TbRuleEngineQueueConsumerManager(ruleEngineConsumerContext, queueKey);
consumerManager = TbRuleEngineQueueConsumerManager.create()
.ctx(ruleEngineConsumerContext)
.queueKey(queueKey)
.consumerExecutor(consumersExecutor)
.scheduler(scheduler)
.taskExecutor(mgmtExecutor)
.build();
}
@After
public void afterEach() {
consumerManager.stop();
consumerManager.awaitStop();
ruleEngineConsumerContext.stop();
consumersExecutor.shutdownNow();
scheduler.shutdownNow();
mgmtExecutor.shutdownNow();
if (generateQueueMsgs) {
await().atMost(10, TimeUnit.SECONDS)
@ -199,14 +215,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
Set<TopicPartitionInfo> partitions = createTpis(2, 3, 4);
consumerManager.update(partitions);
partitions = createTpis(3, 4, 5);
consumerManager.update(partitions);
partitions = createTpis(1, 2, 3);
consumerManager.update(partitions);
// simulated multiple partition change events before consumer is ready; only latest partitions should be processed
verifyNoInteractions(queueFactory);
ruleEngineConsumerContext.setReady(true);
await().atMost(2, TimeUnit.SECONDS)
.until(() -> consumers.size() == 3);
for (TopicPartitionInfo partition : partitions) {
@ -222,14 +231,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
Set<TopicPartitionInfo> partitions = createTpis(2, 3, 4);
consumerManager.update(partitions);
partitions = createTpis(3, 4, 5);
consumerManager.update(partitions);
partitions = createTpis(1, 2, 3);
consumerManager.update(partitions);
verifyNoInteractions(queueFactory);
ruleEngineConsumerContext.setReady(true);
await().atMost(2, TimeUnit.SECONDS)
.until(() -> consumers.size() == 1);
TestConsumer consumer = getConsumer();
@ -240,7 +242,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testPartitionsUpdate_singleConsumer() {
queue.setConsumerPerPartition(false);
consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = Collections.emptySet();
consumerManager.update(partitions);
@ -273,7 +274,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testPartitionsUpdate_consumerPerPartition() {
queue.setConsumerPerPartition(true);
consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
consumerManager.update(Collections.emptySet());
verify(queueFactory, after(1000).never()).createToRuleEngineMsgConsumer(any());
@ -316,7 +316,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testConfigUpdate_singleConsumer() {
queue.setConsumerPerPartition(false);
consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3);
consumerManager.update(partitions);
TestConsumer consumer = getConsumer();
@ -342,7 +341,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testConfigUpdate_consumerPerPartition() {
queue.setConsumerPerPartition(true);
consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3);
consumerManager.update(partitions);
TestConsumer consumer1 = getConsumer(1);
@ -375,7 +373,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testConfigUpdate_fromSingleToConsumerPerPartition() {
queue.setConsumerPerPartition(false);
consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3);
consumerManager.update(partitions);
TestConsumer consumer = getConsumer();
@ -395,7 +392,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testConfigUpdate_fromConsumerPerPartitionToSingle() {
queue.setConsumerPerPartition(true);
consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2, 3);
consumerManager.update(partitions);
TestConsumer consumer1 = getConsumer(1);
@ -419,7 +415,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testStop() {
queue.setConsumerPerPartition(true);
consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
consumerManager.update(createTpis(1));
TestConsumer consumer = getConsumer(1);
verifySubscribedAndLaunched(consumer, 1);
@ -437,7 +432,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testDelete_consumerPerPartition() {
queue.setConsumerPerPartition(true);
consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2);
consumerManager.update(partitions);
TestConsumer consumer1 = getConsumer(1);
@ -459,7 +453,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
int msgCount = totalConsumedMsgs.get();
await().atLeast(2, TimeUnit.SECONDS) // based on topicDeletionDelayInSec(5) = 5 - ( 3 seconds the code may execute starting consumerManager.delete() call)
.atMost(7, TimeUnit.SECONDS)
.atMost(20, TimeUnit.SECONDS)
.untilAsserted(() -> {
partitions.stream()
.map(TopicPartitionInfo::getFullTopicName)
@ -481,7 +475,6 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testDelete_singleConsumer() {
queue.setConsumerPerPartition(false);
consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Set<TopicPartitionInfo> partitions = createTpis(1, 2);
consumerManager.update(partitions);
TestConsumer consumer = getConsumer();
@ -499,7 +492,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
int msgCount = totalConsumedMsgs.get();
await().atLeast(2, TimeUnit.SECONDS) // based on topicDeletionDelayInSec(5) = 5 - ( 3 seconds the code may execute starting consumerManager.delete() call)
.atMost(7, TimeUnit.SECONDS)
.atMost(20, TimeUnit.SECONDS)
.untilAsserted(() -> {
partitions.stream()
.map(TopicPartitionInfo::getFullTopicName)
@ -520,10 +513,9 @@ public class TbRuleEngineQueueConsumerManagerTest {
public void testManyDifferentUpdates() throws Exception {
queue.setConsumerPerPartition(RandomUtils.nextBoolean());
consumerManager.init(queue);
ruleEngineConsumerContext.setReady(true);
Supplier<Queue> queueConfigUpdater = () -> {
Queue oldConfig = consumerManager.getQueue();
Queue oldConfig = consumerManager.getConfig();
Queue newConfig = JacksonUtil.clone(oldConfig);
newConfig.setConsumerPerPartition(RandomUtils.nextBoolean());
newConfig.setPollInterval(RandomUtils.nextInt(100, 501));
@ -571,7 +563,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
Set<TopicPartitionInfo> expectedPartitions = latestPartitions;
await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
assertThat(consumerManager.getQueue()).isEqualTo(expectedConfig);
assertThat(consumerManager.getConfig()).isEqualTo(expectedConfig);
assertThat(consumerManager.getPartitions()).isEqualTo(expectedPartitions);
});
@ -640,7 +632,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
}
private void verifyMsgProcessed(TbMsg tbMsg) {
await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> {
await().atMost(15, TimeUnit.SECONDS).untilAsserted(() -> {
verify(actorContext, atLeastOnce()).tell(argThat(msg -> {
return ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(tbMsg.getId());
}));

35
application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java

@ -16,16 +16,14 @@
package org.thingsboard.server.service.security.auth;
import io.jsonwebtoken.Claims;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.notification.NotificationDeliveryMethod;
import org.thingsboard.server.common.data.notification.targets.platform.SystemAdministratorsFilter;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.common.data.security.model.JwtToken;
@ -38,6 +36,8 @@ import org.thingsboard.server.service.security.model.UserPrincipal;
import org.thingsboard.server.service.security.model.token.AccessJwtToken;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Calendar;
import java.util.Date;
import java.util.Optional;
@ -45,19 +45,13 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class JwtTokenFactoryTest {
private JwtTokenFactory tokenFactory;
private AdminSettingsService adminSettingsService;
private NotificationCenter notificationCenter;
private JwtSettingsService jwtSettingsService;
private JwtSettings jwtSettings;
@ -66,12 +60,11 @@ public class JwtTokenFactoryTest {
public void beforeEach() {
jwtSettings = new JwtSettings();
jwtSettings.setTokenIssuer("tb");
jwtSettings.setTokenSigningKey("abewafaf");
jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8)));
jwtSettings.setTokenExpirationTime((int) TimeUnit.HOURS.toSeconds(2));
jwtSettings.setRefreshTokenExpTime((int) TimeUnit.DAYS.toSeconds(7));
adminSettingsService = mock(AdminSettingsService.class);
notificationCenter = mock(NotificationCenter.class);
jwtSettingsService = mockJwtSettingsService();
mockJwtSettings(jwtSettings);
@ -169,21 +162,6 @@ public class JwtTokenFactoryTest {
});
}
@Test
public void testJwtSigningKeyIssueNotification() {
JwtSettings badJwtSettings = jwtSettings;
badJwtSettings.setTokenSigningKey(JwtSettingsService.TOKEN_SIGNING_KEY_DEFAULT);
mockJwtSettings(badJwtSettings);
jwtSettingsService = mockJwtSettingsService();
for (int i = 0; i < 5; i++) { // to check if notification is not sent twice
jwtSettingsService.getJwtSettings();
}
verify(notificationCenter, times(1)).sendGeneralWebNotification(eq(TenantId.SYS_TENANT_ID),
isA(SystemAdministratorsFilter.class), argThat(template -> template.getConfiguration().getDeliveryMethodsTemplates().get(NotificationDeliveryMethod.WEB)
.getBody().contains("The platform is configured to use default JWT Signing Key")));
}
private void mockJwtSettings(JwtSettings settings) {
AdminSettings adminJwtSettings = new AdminSettings();
adminJwtSettings.setJsonValue(JacksonUtil.valueToTree(settings));
@ -192,12 +170,11 @@ public class JwtTokenFactoryTest {
}
private DefaultJwtSettingsService mockJwtSettingsService() {
return new DefaultJwtSettingsService(adminSettingsService, Optional.empty(),
Optional.of(notificationCenter), new DefaultJwtSettingsValidator());
return new DefaultJwtSettingsService(adminSettingsService, Optional.empty(), new DefaultJwtSettingsValidator(), Optional.empty());
}
private void checkExpirationTime(JwtToken jwtToken, int tokenLifetime) {
Claims claims = tokenFactory.parseTokenClaims(jwtToken.getToken()).getBody();
Claims claims = tokenFactory.parseTokenClaims(jwtToken.getToken()).getPayload();
assertThat(claims.getExpiration()).matches(actualExpirationTime -> {
Calendar expirationTime = Calendar.getInstance();
expirationTime.setTime(new Date());

17
application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java

@ -532,20 +532,19 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest {
@Test
public void testExportImportDeviceProfile_betweenTenants_findExistingByName() throws Exception {
public void testExportImportDefaultDeviceProfile_betweenTenants_findExisting() throws Exception {
DeviceProfile defaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId1);
defaultDeviceProfile.setName("non-default-name");
deviceProfileService.saveDeviceProfile(defaultDeviceProfile);
EntityExportData<DeviceProfile> deviceProfileExportData = exportEntity(tenantAdmin1, defaultDeviceProfile.getId());
assertThatThrownBy(() -> {
importEntity(tenantAdmin2, deviceProfileExportData, EntityImportSettings.builder()
.findExistingByName(false)
.build());
}).hasMessageContaining("default device profile is present");
importEntity(tenantAdmin2, deviceProfileExportData, EntityImportSettings.builder()
.findExistingByName(true)
.findExistingByName(false)
.build());
checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, deviceProfileService.findDefaultDeviceProfile(tenantId2));
DeviceProfile importedDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId2);
assertThat(importedDeviceProfile.isDefault()).isTrue();
assertThat(importedDeviceProfile.getName()).isEqualTo(defaultDeviceProfile.getName());
checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, importedDeviceProfile);
}

5
common/actor/pom.xml

@ -65,11 +65,6 @@
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>

69
common/actor/src/test/java/org/thingsboard/server/actors/ActorSystemTest.java

@ -16,12 +16,11 @@
package org.thingsboard.server.actors;
import lombok.extern.slf4j.Slf4j;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.id.DeviceId;
@ -37,29 +36,31 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import static org.assertj.core.api.Assertions.assertThat;
@Slf4j
@RunWith(MockitoJUnitRunner.class)
public class ActorSystemTest {
public static final String ROOT_DISPATCHER = "root-dispatcher";
private static final int _100K = 100 * 1024;
public static final int TIMEOUT_AWAIT_MAX_SEC = 100;
public static final int TIMEOUT_AWAIT_MAX_SEC = 30;
private volatile TbActorSystem actorSystem;
private volatile ExecutorService submitPool;
private ExecutorService executor;
private int parallelism;
@Before
@BeforeEach
public void initActorSystem() {
int cores = Runtime.getRuntime().availableProcessors();
parallelism = Math.max(2, cores / 2);
log.debug("parallelism {}", parallelism);
TbActorSystemSettings settings = new TbActorSystemSettings(5, parallelism, 42);
actorSystem = new DefaultTbActorSystem(settings);
submitPool = Executors.newFixedThreadPool(parallelism, ThingsBoardThreadFactory.forName(getClass().getSimpleName() + "-submit-test-scope")); //order guaranteed
}
@After
@AfterEach
public void shutdownActorSystem() {
actorSystem.stop();
submitPool.shutdownNow();
@ -126,8 +127,8 @@ public class ActorSystemTest {
actorId2.tell(new IntTbActorMsg(42));
actorSystem.stop(actorId1);
Assert.assertTrue(testCtx2.getLatch().await(1, TimeUnit.SECONDS));
Assert.assertFalse(testCtx1.getLatch().await(1, TimeUnit.SECONDS));
Assertions.assertTrue(testCtx2.getLatch().await(1, TimeUnit.SECONDS));
Assertions.assertFalse(testCtx1.getLatch().await(1, TimeUnit.SECONDS));
}
@Test
@ -136,25 +137,39 @@ public class ActorSystemTest {
actorSystem.createDispatcher(ROOT_DISPATCHER, executor);
ActorTestCtx testCtx1 = getActorTestCtx(1);
ActorTestCtx testCtx2 = getActorTestCtx(1);
assertThat(testCtx1.getLatch().getCount()).as("testCtx1 latch initial state").isEqualTo(1);
assertThat(testCtx2.getLatch().getCount()).as("testCtx2 latch initial state").isEqualTo(1);
TbActorId actorId = new TbEntityActorId(new DeviceId(UUID.randomUUID()));
final CountDownLatch initLatch = new CountDownLatch(1);
final CountDownLatch actorsReadyLatch = new CountDownLatch(2);
submitPool.submit(() -> {
log.info("submit 1");
actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx1, initLatch));
actorsReadyLatch.countDown();
log.info("done 1");
});
submitPool.submit(() -> {
log.info("submit 2");
actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx2, initLatch));
actorsReadyLatch.countDown();
log.info("done 2");
});
initLatch.countDown(); //replacement for Thread.wait(500) in the SlowCreateActorCreator
Assert.assertTrue(actorsReadyLatch.await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS));
Assertions.assertTrue(actorsReadyLatch.await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS));
log.info("actorsReadyLatch ok");
actorSystem.tell(actorId, new IntTbActorMsg(42));
Assert.assertTrue(testCtx1.getLatch().await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS));
Assert.assertFalse(testCtx2.getLatch().await(1, TimeUnit.SECONDS));
//only one of two contexts are initialized. no matter Ctx1 or Ctx2
Awaitility.await("one of two actors latch zeroed").atMost(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS)
.until(() -> testCtx1.getLatch().getCount() + testCtx2.getLatch().getCount() == 1);
Thread.yield();
if (testCtx1.getLatch().getCount() == 0) {
assertThat(testCtx2.getLatch().await(100, TimeUnit.MILLISECONDS)).as("testCtx2 never latched").isFalse();
} else {
assertThat(testCtx1.getLatch().await(100, TimeUnit.MILLISECONDS)).as("testCtx1 never latched").isFalse();
}
}
@Test
@ -173,13 +188,13 @@ public class ActorSystemTest {
});
}
initLatch.countDown();
Assert.assertTrue(actorsReadyLatch.await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS));
Assertions.assertTrue(actorsReadyLatch.await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS));
actorSystem.tell(actorId, new IntTbActorMsg(42));
Assert.assertTrue(testCtx.getLatch().await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS));
Assertions.assertTrue(testCtx.getLatch().await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS));
//One for creation and one for message
Assert.assertEquals(2, testCtx.getInvocationCount().get());
Assertions.assertEquals(2, testCtx.getInvocationCount().get());
}
@Test
@ -197,9 +212,9 @@ public class ActorSystemTest {
actorId1.tell(new IntTbActorMsg(42));
actorId2.tell(new IntTbActorMsg(42));
Assert.assertFalse(testCtx1.getLatch().await(2, TimeUnit.SECONDS));
Assert.assertTrue(testCtx2.getLatch().await(1, TimeUnit.SECONDS));
Assert.assertTrue(testCtx1.getLatch().await(3, TimeUnit.SECONDS));
Assertions.assertFalse(testCtx1.getLatch().await(2, TimeUnit.SECONDS));
Assertions.assertTrue(testCtx2.getLatch().await(1, TimeUnit.SECONDS));
Assertions.assertTrue(testCtx1.getLatch().await(3, TimeUnit.SECONDS));
}
@ -233,16 +248,16 @@ public class ActorSystemTest {
log.info("Submitted all messages");
testCtxes.forEach(ctx -> {
try {
boolean success = ctx.getLatch().await(1, TimeUnit.MINUTES);
boolean success = ctx.getLatch().await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS);
if (!success) {
log.warn("Failed: {}, {}", ctx.getActual().get(), ctx.getInvocationCount().get());
}
Assert.assertTrue(success);
Assert.assertEquals(expected, ctx.getActual().get());
Assert.assertEquals(msgNumber, ctx.getInvocationCount().get());
Assertions.assertTrue(success);
Assertions.assertEquals(expected, ctx.getActual().get());
Assertions.assertEquals(msgNumber, ctx.getInvocationCount().get());
ctx.clear();
} catch (InterruptedException e) {
e.printStackTrace();
log.error("interrupted", e);
}
});
long duration = System.nanoTime() - start;

9
common/actor/src/test/java/org/thingsboard/server/actors/SlowCreateActor.java

@ -28,11 +28,13 @@ public class SlowCreateActor extends TestRootActor {
public SlowCreateActor(TbActorId actorId, ActorTestCtx testCtx, CountDownLatch initLatch) {
super(actorId, testCtx);
try {
log.info("awaiting on latch {} ...", initLatch);
initLatch.await(TIMEOUT_AWAIT_MAX_MS, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
log.error("interrupted", e);
}
testCtx.getInvocationCount().incrementAndGet();
log.info("finished {} ...", initLatch);
}
public static class SlowCreateActorCreator implements TbActorCreator {
@ -54,7 +56,10 @@ public class SlowCreateActor extends TestRootActor {
@Override
public TbActor createActor() {
return new SlowCreateActor(actorId, testCtx, initLatch);
log.info("creating slow actor...");
SlowCreateActor slowCreateActor = new SlowCreateActor(actorId, testCtx, initLatch);
log.info("created slow actor {}", slowCreateActor);
return slowCreateActor;
}
}
}

3
common/actor/src/test/resources/junit-platform.properties

@ -0,0 +1,3 @@
junit.jupiter.execution.parallel.enabled = true
junit.jupiter.execution.parallel.mode.default = concurrent
junit.jupiter.execution.parallel.mode.classes.default = concurrent

0
common/actor/src/test/resources/logback.xml → common/actor/src/test/resources/logback-test.xml

5
common/cache/pom.xml

@ -97,11 +97,6 @@
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>

21
common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCacheEvictEvent.java

@ -0,0 +1,21 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.customer;
import org.thingsboard.server.common.data.id.TenantId;
public record CustomerCacheEvictEvent(TenantId tenantId, String newTitle, String oldTitle) {
}

42
common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCacheKey.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.customer;
import lombok.EqualsAndHashCode;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.thingsboard.server.common.data.id.TenantId;
import java.io.Serial;
import java.io.Serializable;
@EqualsAndHashCode
@RequiredArgsConstructor
public class CustomerCacheKey implements Serializable {
@Serial
private static final long serialVersionUID = 5706958428811356925L;
@NonNull
private final TenantId tenantId;
private final String title;
@Override
public String toString() {
return tenantId.getId() + "_" + title;
}
}

33
common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerCaffeineCache.java

@ -0,0 +1,33 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.customer;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CaffeineTbTransactionalCache;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.Customer;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true)
@Service("CustomerCache")
public class CustomerCaffeineCache extends CaffeineTbTransactionalCache<CustomerCacheKey, Customer> {
public CustomerCaffeineCache(CacheManager cacheManager) {
super(cacheManager, CacheConstants.CUSTOMER_CACHE);
}
}

35
common/cache/src/main/java/org/thingsboard/server/cache/customer/CustomerRedisCache.java

@ -0,0 +1,35 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.customer;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CacheSpecsMap;
import org.thingsboard.server.cache.RedisTbTransactionalCache;
import org.thingsboard.server.cache.TBRedisCacheConfiguration;
import org.thingsboard.server.cache.TbJsonRedisSerializer;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.Customer;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis")
@Service("CustomerCache")
public class CustomerRedisCache extends RedisTbTransactionalCache<CustomerCacheKey, Customer> {
public CustomerRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) {
super(CacheConstants.CUSTOMER_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbJsonRedisSerializer<>(Customer.class));
}
}

11
common/cache/src/main/java/org/thingsboard/server/cache/limits/DefaultRateLimitService.java

@ -63,12 +63,21 @@ public class DefaultRateLimitService implements RateLimitService {
@Override
public boolean checkRateLimit(LimitedApi api, TenantId tenantId, Object level) {
return checkRateLimit(api, tenantId, level, false);
}
@Override
public boolean checkRateLimit(LimitedApi api, TenantId tenantId, Object level, boolean ignoreTenantNotFound) {
if (tenantId.isSysTenantId()) {
return true;
}
TenantProfile tenantProfile = tenantProfileProvider.get(tenantId);
if (tenantProfile == null) {
throw new TenantProfileNotFoundException(tenantId);
if (ignoreTenantNotFound) {
return true;
} else {
throw new TenantProfileNotFoundException(tenantId);
}
}
String rateLimitConfig = tenantProfile.getProfileConfiguration()

2
common/cache/src/main/java/org/thingsboard/server/cache/limits/RateLimitService.java

@ -24,6 +24,8 @@ public interface RateLimitService {
boolean checkRateLimit(LimitedApi api, TenantId tenantId, Object level);
boolean checkRateLimit(LimitedApi api, TenantId tenantId, Object level, boolean ignoreTenantNotFound);
boolean checkRateLimit(LimitedApi api, Object level, String rateLimitConfig);
void cleanUp(LimitedApi api, Object level);

21
common/cache/src/main/java/org/thingsboard/server/cache/user/UserCacheEvictEvent.java

@ -0,0 +1,21 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.user;
import org.thingsboard.server.common.data.id.TenantId;
public record UserCacheEvictEvent(TenantId tenantId, String newEmail, String oldEmail) {
}

42
common/cache/src/main/java/org/thingsboard/server/cache/user/UserCacheKey.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.user;
import lombok.EqualsAndHashCode;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.thingsboard.server.common.data.id.TenantId;
import java.io.Serial;
import java.io.Serializable;
@EqualsAndHashCode
@RequiredArgsConstructor
public class UserCacheKey implements Serializable {
@Serial
private static final long serialVersionUID = 7357353074893750678L;
@NonNull
private final TenantId tenantId;
private final String email;
@Override
public String toString() {
return tenantId.getId() + "_" + email;
}
}

33
common/cache/src/main/java/org/thingsboard/server/cache/user/UserCaffeineCache.java

@ -0,0 +1,33 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.user;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CaffeineTbTransactionalCache;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.User;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true)
@Service("UserCache")
public class UserCaffeineCache extends CaffeineTbTransactionalCache<UserCacheKey, User> {
public UserCaffeineCache(CacheManager cacheManager) {
super(cacheManager, CacheConstants.USER_CACHE);
}
}

35
common/cache/src/main/java/org/thingsboard/server/cache/user/UserRedisCache.java

@ -0,0 +1,35 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.user;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CacheSpecsMap;
import org.thingsboard.server.cache.RedisTbTransactionalCache;
import org.thingsboard.server.cache.TBRedisCacheConfiguration;
import org.thingsboard.server.cache.TbJsonRedisSerializer;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.User;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis")
@Service("UserCache")
public class UserRedisCache extends RedisTbTransactionalCache<UserCacheKey, User> {
public UserRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) {
super(CacheConstants.USER_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbJsonRedisSerializer<>(User.class));
}
}

5
common/cache/src/test/java/org/thingsboard/server/cache/CacheSpecsMapTest.java

@ -26,6 +26,7 @@ import org.springframework.cache.support.SimpleCacheManager;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.thingsboard.server.common.data.CacheConstants;
import static org.assertj.core.api.Assertions.assertThat;
@ -53,8 +54,8 @@ public class CacheSpecsMapTest {
@Test
public void givenCacheConfig_whenCacheManagerReady_thenVerifyExistedCachesWithNoTransactionAwareCacheDecorator() {
// We no longer use built-in transaction support for the caches, because we have our own cache cleanup and transaction logic that implements CAS.
assertThat(cacheManager.getCache("relations")).isInstanceOf(CaffeineCache.class);
assertThat(cacheManager.getCache("devices")).isInstanceOf(CaffeineCache.class);
assertThat(cacheManager.getCache(CacheConstants.RELATIONS_CACHE)).isInstanceOf(CaffeineCache.class);
assertThat(cacheManager.getCache(CacheConstants.DEVICE_CACHE)).isInstanceOf(CaffeineCache.class);
}
@Test

3
common/cache/src/test/resources/junit-platform.properties

@ -0,0 +1,3 @@
junit.jupiter.execution.parallel.enabled = true
junit.jupiter.execution.parallel.mode.default = concurrent
junit.jupiter.execution.parallel.mode.classes.default = concurrent

0
common/cache/src/test/resources/logback.xml → common/cache/src/test/resources/logback-test.xml

5
common/cluster-api/pom.xml

@ -109,11 +109,6 @@
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>

5
common/dao-api/pom.xml

@ -105,11 +105,6 @@
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>

2
common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java

@ -41,6 +41,8 @@ public interface AssetService extends EntityDaoService {
Asset findAssetByTenantIdAndName(TenantId tenantId, String name);
ListenableFuture<Asset> findAssetByTenantIdAndNameAsync(TenantId tenantId, String name);
Asset saveAsset(Asset asset, boolean doValidate);
Asset saveAsset(Asset asset);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/customer/CustomerService.java

@ -31,6 +31,8 @@ public interface CustomerService extends EntityDaoService {
Optional<Customer> findCustomerByTenantIdAndTitle(TenantId tenantId, String title);
ListenableFuture<Optional<Customer>> findCustomerByTenantIdAndTitleAsync(TenantId tenantId, String title);
ListenableFuture<Customer> findCustomerByIdAsync(TenantId tenantId, CustomerId customerId);
Customer saveCustomer(Customer customer);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java

@ -72,6 +72,8 @@ public interface DashboardService extends EntityDaoService {
DashboardInfo findFirstDashboardInfoByTenantIdAndName(TenantId tenantId, String name);
ListenableFuture<DashboardInfo> findFirstDashboardInfoByTenantIdAndNameAsync(TenantId tenantId, String name);
List<Dashboard> findTenantDashboardsByTitle(TenantId tenantId, String title);
}

2
common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java

@ -49,6 +49,8 @@ public interface DeviceService extends EntityDaoService {
Device findDeviceByTenantIdAndName(TenantId tenantId, String name);
ListenableFuture<Device> findDeviceByTenantIdAndNameAsync(TenantId tenantId, String name);
Device saveDevice(Device device);
Device saveDevice(Device device, boolean doValidate);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java

@ -42,6 +42,8 @@ public interface EdgeService extends EntityDaoService {
Edge findEdgeByTenantIdAndName(TenantId tenantId, String name);
ListenableFuture<Edge> findEdgeByTenantIdAndNameAsync(TenantId tenantId, String name);
Optional<Edge> findEdgeByRoutingKey(TenantId tenantId, String routingKey);
Edge saveEdge(Edge edge);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java

@ -54,6 +54,8 @@ public interface EntityViewService extends EntityDaoService {
EntityView findEntityViewByTenantIdAndName(TenantId tenantId, String name);
ListenableFuture<EntityView> findEntityViewByTenantIdAndNameAsync(TenantId tenantId, String name);
PageData<EntityView> findEntityViewByTenantId(TenantId tenantId, PageLink pageLink);
PageData<EntityViewInfo> findEntityViewInfosByTenantId(TenantId tenantId, PageLink pageLink);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java

@ -41,6 +41,8 @@ public interface UserService extends EntityDaoService {
User findUserByTenantIdAndEmail(TenantId tenantId, String email);
ListenableFuture<User> findUserByTenantIdAndEmailAsync(TenantId tenantId, String email);
User saveUser(TenantId tenantId, User user);
UserCredentials findUserCredentialsByUserId(TenantId tenantId, UserId userId);

5
common/data/pom.xml

@ -73,11 +73,6 @@
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>

2
common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java

@ -21,6 +21,8 @@ public class CacheConstants {
public static final String DEVICE_CACHE = "devices";
public static final String SESSIONS_CACHE = "sessions";
public static final String ASSET_CACHE = "assets";
public static final String CUSTOMER_CACHE = "customers";
public static final String USER_CACHE = "users";
public static final String ENTITY_VIEW_CACHE = "entityViews";
public static final String EDGE_CACHE = "edges";
public static final String CLAIM_DEVICES_CACHE = "claimDevices";

4
common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java

@ -18,6 +18,7 @@ package org.thingsboard.server.common.data;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.core.JsonProcessingException;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@ -33,7 +34,6 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.validation.Length;
import org.thingsboard.server.common.data.validation.NoXss;
import jakarta.validation.Valid;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@ -42,7 +42,7 @@ import java.io.IOException;
@ToString(exclude = {"image", "profileDataBytes"})
@EqualsAndHashCode(callSuper = true)
@Slf4j
public class DeviceProfile extends BaseData<DeviceProfileId> implements HasName, HasTenantId, HasOtaPackage, HasRuleEngineProfile, ExportableEntity<DeviceProfileId>, HasImage {
public class DeviceProfile extends BaseData<DeviceProfileId> implements HasName, HasTenantId, HasOtaPackage, HasRuleEngineProfile, ExportableEntity<DeviceProfileId>, HasImage, HasDefaultOption {
private static final long serialVersionUID = 6998485460273302018L;

13
netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/IntegrationTestSuite.java → common/data/src/main/java/org/thingsboard/server/common/data/HasDefaultOption.java

@ -13,15 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.mqtt.integration;
package org.thingsboard.server.common.data;
import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith;
public interface HasDefaultOption {
@RunWith(ClasspathSuite.class)
@ClasspathSuite.ClassnameFilters({
"org.thingsboard.mqtt.integration.*Test",
})
public class IntegrationTestSuite {
boolean isDefault();
}
}

3
common/data/src/main/java/org/thingsboard/server/common/data/asset/AssetProfile.java

@ -22,6 +22,7 @@ import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.BaseData;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasDefaultOption;
import org.thingsboard.server.common.data.HasImage;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HasRuleEngineProfile;
@ -38,7 +39,7 @@ import org.thingsboard.server.common.data.validation.NoXss;
@ToString(exclude = {"image"})
@EqualsAndHashCode(callSuper = true)
@Slf4j
public class AssetProfile extends BaseData<AssetProfileId> implements HasName, HasTenantId, HasRuleEngineProfile, ExportableEntity<AssetProfileId>, HasImage {
public class AssetProfile extends BaseData<AssetProfileId> implements HasName, HasTenantId, HasRuleEngineProfile, ExportableEntity<AssetProfileId>, HasImage, HasDefaultOption {
private static final long serialVersionUID = 6998485460273302018L;

1
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/lwm2m/OtherConfiguration.java

@ -39,4 +39,5 @@ public class OtherConfiguration extends PowerSavingConfiguration {
private Long pagingTransmissionWindow;
private String fwUpdateResource;
private String swUpdateResource;
private String defaultObjectIDVer;
}

30
common/data/src/main/java/org/thingsboard/server/common/data/exception/RateLimitExceededException.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.exception;
import org.thingsboard.server.common.data.limit.LimitedApi;
public class RateLimitExceededException extends AbstractRateLimitException {
public RateLimitExceededException(String message) {
super(message);
}
public RateLimitExceededException(LimitedApi api) {
super("Rate limit for " + api.getLabel() + " is exceeded");
}
}

3
common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java

@ -39,7 +39,8 @@ public enum LimitedApi {
TWO_FA_VERIFICATION_CODE_SEND(false, true),
TWO_FA_VERIFICATION_CODE_CHECK(false, true),
TRANSPORT_MESSAGES_PER_TENANT("transport messages", true),
TRANSPORT_MESSAGES_PER_DEVICE("transport messages per device", false);
TRANSPORT_MESSAGES_PER_DEVICE("transport messages per device", false),
EMAILS("emails sending", true);
private Function<DefaultTenantProfileConfiguration, String> configExtractor;
@Getter

2
common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java

@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.validation.NoXss;
import java.util.Optional;
@Data
public class Queue extends BaseDataWithAdditionalInfo<QueueId> implements HasName, HasTenantId {
public class Queue extends BaseDataWithAdditionalInfo<QueueId> implements HasName, HasTenantId, QueueConfig {
private TenantId tenantId;
@NoXss
@Length(fieldName = "name")

12
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntityContainer.java → common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java

@ -13,16 +13,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.util;
package org.thingsboard.server.common.data.queue;
import lombok.Data;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.EntityId;
public interface QueueConfig {
@Data
public class EntityContainer {
boolean isConsumerPerPartition();
private EntityId entityId;
private EntityType entityType;
int getPollInterval();
}

9
common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java

@ -23,6 +23,7 @@ import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasDefaultOption;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.id.RuleChainId;
@ -35,7 +36,7 @@ import org.thingsboard.server.common.data.validation.NoXss;
@Data
@EqualsAndHashCode(callSuper = true)
@Slf4j
public class RuleChain extends BaseDataWithAdditionalInfo<RuleChainId> implements HasName, HasTenantId, ExportableEntity<RuleChainId> {
public class RuleChain extends BaseDataWithAdditionalInfo<RuleChainId> implements HasName, HasTenantId, ExportableEntity<RuleChainId>, HasDefaultOption {
private static final long serialVersionUID = -5656679015121935465L;
@ -108,4 +109,10 @@ public class RuleChain extends BaseDataWithAdditionalInfo<RuleChainId> implement
setJson(data, json -> this.configuration = json, bytes -> this.configurationBytes = bytes);
}
@JsonIgnore
@Override
public boolean isDefault() {
return root && type == RuleChainType.CORE;
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtSettings.java

@ -48,7 +48,7 @@ public class JwtSettings {
* Key is used to sign {@link JwtToken}.
* Base64 encoded
*/
@Schema(description = "The JWT key is used to sing token. Base64 encoded.", example = "cTU4WnNqemI2aU5wbWVjdm1vYXRzanhjNHRUcXliMjE=")
@Schema(description = "The JWT key is used to sing token. Base64 encoded.", example = "dkVTUzU2M2VMWUNwVVltTUhQU2o5SUM0Tkc3M0k2Ykdwcm85QTl6R0RaQ252OFlmVDk2OEptZXBNcndGeExFZg==")
private String tokenSigningKey;
}

14
common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java

@ -19,18 +19,18 @@ import com.github.os72.protobuf.dynamic.DynamicSchema;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List;
import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@RunWith(MockitoJUnitRunner.class)
@ExtendWith(MockitoExtension.class)
public class DynamicProtoUtilsTest {
@Test

17
common/data/src/test/java/org/thingsboard/server/common/data/UUIDConverterTest.java

@ -16,11 +16,10 @@
package org.thingsboard.server.common.data;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import org.junit.Assert;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Assertions;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Random;
import java.util.UUID;
@ -28,14 +27,14 @@ import java.util.UUID;
/**
* Created by ashvayka on 14.07.17.
*/
@RunWith(MockitoJUnitRunner.class)
@ExtendWith(MockitoExtension .class)
public class UUIDConverterTest {
@Test
public void basicUuidToStringTest() {
UUID original = UUID.fromString("58e0a7d7-eebc-11d8-9669-0800200c9a66");
String result = UUIDConverter.fromTimeUUID(original);
Assert.assertEquals("1d8eebc58e0a7d796690800200c9a66", result);
Assertions.assertEquals("1d8eebc58e0a7d796690800200c9a66", result);
}
@ -47,13 +46,13 @@ public class UUIDConverterTest {
@Test
public void basicUuidConversion() {
UUID original = UUID.fromString("3dd11790-abf2-11ea-b151-83a091b9d4cc");
Assert.assertEquals(Uuids.unixTimestamp(original), 1591886749577L);
Assertions.assertEquals(Uuids.unixTimestamp(original), 1591886749577L);
}
@Test
public void basicStringToUUIDTest() {
UUID result = UUIDConverter.fromString("1d8eebc58e0a7d796690800200c9a66");
Assert.assertEquals(UUID.fromString("58e0a7d7-eebc-11d8-9669-0800200c9a66"), result);
Assertions.assertEquals(UUID.fromString("58e0a7d7-eebc-11d8-9669-0800200c9a66"), result);
}
@Test
@ -83,7 +82,7 @@ public class UUIDConverterTest {
System.out.println("Before: " + before + " | " + beforeStr);
System.out.println("After: " + after + " | " + afterStr);
}
Assert.assertTrue(afterStr.compareTo(beforeStr) >= 0);
Assertions.assertTrue(afterStr.compareTo(beforeStr) >= 0);
}
}

6
common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java

@ -15,14 +15,14 @@
*/
package org.thingsboard.server.common.data.id;
import org.junit.Assert;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class EntityIdTest {
@Test
public void givenConstantNullUuid_whenCompare_thenToStringEqualsPredefinedUuid() {
Assert.assertEquals("13814000-1dd2-11b2-8080-808080808080", EntityId.NULL_UUID.toString());
Assertions.assertEquals("13814000-1dd2-11b2-8080-808080808080", EntityId.NULL_UUID.toString());
}
}

3
common/data/src/test/resources/junit-platform.properties

@ -0,0 +1,3 @@
junit.jupiter.execution.parallel.enabled = true
junit.jupiter.execution.parallel.mode.default = concurrent
junit.jupiter.execution.parallel.mode.classes.default = concurrent

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save