Browse Source

Merge remote-tracking branch 'upstream/rc' into bug/map/switch-datasource-type-disable

pull/13057/head
Vladyslav_Prykhodko 2 years ago
parent
commit
ae41636d83
  1. 4
      application/src/main/data/json/system/widget_types/bar_chart.json
  2. 4
      application/src/main/data/json/system/widget_types/bar_chart_with_labels.json
  3. 4
      application/src/main/data/json/system/widget_types/bars.json
  4. 4
      application/src/main/data/json/system/widget_types/doughnut.json
  5. 4
      application/src/main/data/json/system/widget_types/horizontal_doughnut.json
  6. 4
      application/src/main/data/json/system/widget_types/image_map.json
  7. 4
      application/src/main/data/json/system/widget_types/line_chart.json
  8. 4
      application/src/main/data/json/system/widget_types/map.json
  9. 4
      application/src/main/data/json/system/widget_types/pie.json
  10. 4
      application/src/main/data/json/system/widget_types/point_chart.json
  11. 4
      application/src/main/data/json/system/widget_types/polar_area.json
  12. 4
      application/src/main/data/json/system/widget_types/radar.json
  13. 4
      application/src/main/data/json/system/widget_types/range_chart.json
  14. 4
      application/src/main/data/json/system/widget_types/route_map.json
  15. 4
      application/src/main/data/json/system/widget_types/state_chart.json
  16. 4
      application/src/main/data/json/system/widget_types/time_series_chart.json
  17. 4
      application/src/main/data/json/system/widget_types/trip_map.json
  18. 2
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java
  19. 5
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java
  20. 12
      application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java
  21. 3
      application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java
  22. 18
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java
  23. 18
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  24. 43
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractPartitionBasedConsumerService.java
  25. 6
      application/src/main/resources/thingsboard.yml
  26. 3
      application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java
  27. 2
      application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java
  28. 19
      application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java
  29. 2
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java
  30. 6
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java
  31. 6
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java
  32. 5
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java
  33. 5
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java
  34. 6
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java
  35. 15
      common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java
  36. 3
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java
  37. 3
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java
  38. 3
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java
  39. 30
      common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java
  40. 50
      common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java
  41. 5
      common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java
  42. 21
      common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java
  43. 4
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java
  44. 12
      common/message/src/test/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfoTest.java
  45. 14
      common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java
  46. 9
      common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java
  47. 5
      common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java
  48. 16
      common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java
  49. 9
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java
  50. 2
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java
  51. 4
      common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java
  52. 36
      common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java
  53. 8
      common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java
  54. 23
      common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java
  55. 68
      common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java
  56. 14
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java
  57. 46
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java
  58. 3
      common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java
  59. 5
      common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java
  60. 7
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java
  61. 7
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java
  62. 14
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java
  63. 6
      edqs/src/main/resources/edqs.yml
  64. 18
      pom.xml
  65. 9
      ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.html
  66. 7
      ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.scss
  67. 7
      ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts
  68. 1
      ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart-widget.component.html
  69. 9
      ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart.component.html
  70. 7
      ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart.component.scss
  71. 7
      ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart.component.ts
  72. 9
      ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.html
  73. 6
      ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.scss
  74. 7
      ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts
  75. 11
      ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.component.html
  76. 6
      ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.component.scss
  77. 3
      ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.component.ts
  78. 2
      ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts
  79. 7
      ui-ngx/src/app/modules/home/components/widget/lib/maps/data-layer/circles-data-layer.ts
  80. 45
      ui-ngx/src/app/modules/home/components/widget/lib/maps/data-layer/map-data-layer.ts
  81. 7
      ui-ngx/src/app/modules/home/components/widget/lib/maps/data-layer/markers-data-layer.ts
  82. 7
      ui-ngx/src/app/modules/home/components/widget/lib/maps/data-layer/polygons-data-layer.ts
  83. 32
      ui-ngx/src/app/modules/home/components/widget/lib/maps/data-layer/trips-data-layer.ts
  84. 11
      ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget.component.html
  85. 6
      ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget.component.scss
  86. 4
      ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget.component.ts
  87. 7
      ui-ngx/src/app/modules/home/components/widget/lib/maps/map.ts
  88. 10
      ui-ngx/src/app/modules/home/components/widget/lib/maps/panels/map-timeline-panel.component.ts
  89. 11
      ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-widget-settings.component.ts
  90. 2
      ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/widget-action.component.html
  91. 24
      ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/widget-action.component.ts
  92. 37
      ui-ngx/src/app/modules/home/components/widget/lib/settings/common/filter/filter-select.component.html
  93. 4
      ui-ngx/src/app/modules/home/components/widget/lib/settings/common/filter/filter-select.component.ts
  94. 80
      ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-source-row.component.html
  95. 56
      ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-source-row.component.scss
  96. 207
      ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-source-row.component.ts
  97. 43
      ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-sources.component.html
  98. 53
      ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-sources.component.scss
  99. 147
      ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-sources.component.ts
  100. 2
      ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/data-layer-color-settings-panel.component.html

4
application/src/main/data/json/system/widget_types/bar_chart.json

@ -9,9 +9,9 @@
"sizeX": 8,
"sizeY": 5,
"resources": [],
"templateHtml": "<tb-time-series-chart-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-time-series-chart-widget>",
"templateHtml": "<tb-time-series-chart-widget \n [ctx]=\"ctx\">\n</tb-time-series-chart-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'bar',\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('bar'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'bar',\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('bar'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n",
"settingsSchema": "{}",
"dataKeySettingsSchema": "{}",
"latestDataKeySettingsSchema": "{}",

4
application/src/main/data/json/system/widget_types/bar_chart_with_labels.json

@ -9,9 +9,9 @@
"sizeX": 8,
"sizeY": 5,
"resources": [],
"templateHtml": "<tb-bar-chart-with-labels-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-bar-chart-with-labels-widget>",
"templateHtml": "<tb-bar-chart-with-labels-widget \n [ctx]=\"ctx\">\n</tb-bar-chart-with-labels-widget>",
"templateCss": ".legend {\n font-size: 13px;\n line-height: 10px;\n}\n\n.legend table { \n border-spacing: 0px;\n border-collapse: separate;\n}\n\n.mouse-events .flot-overlay {\n cursor: crosshair; \n}\n\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'humidity', label: 'Humidity', type: 'timeseries' }];\n }\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'humidity', label: 'Humidity', type: 'timeseries' }];\n }\n };\n}\n",
"settingsSchema": "{}",
"dataKeySettingsSchema": "{}",
"latestDataKeySettingsSchema": "{}",

4
application/src/main/data/json/system/widget_types/bars.json

@ -9,9 +9,9 @@
"sizeX": 5,
"sizeY": 4,
"resources": [],
"templateHtml": "<tb-bar-chart-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-bar-chart-widget>",
"templateHtml": "<tb-bar-chart-widget \n [ctx]=\"ctx\">\n</tb-bar-chart-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"settingsDirective": "tb-bar-chart-widget-settings",

4
application/src/main/data/json/system/widget_types/doughnut.json

@ -9,9 +9,9 @@
"sizeX": 4,
"sizeY": 3,
"resources": [],
"templateHtml": "<tb-doughnut-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-doughnut-widget>",
"templateHtml": "<tb-doughnut-widget \n [ctx]=\"ctx\">\n</tb-doughnut-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.doughnutWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.doughnutWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '380px',\n previewHeight: '300px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind power', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar power', type: 'timeseries', color: '#FF4D5A' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.doughnutWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.doughnutWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '380px',\n previewHeight: '300px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind power', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar power', type: 'timeseries', color: '#FF4D5A' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"settingsDirective": "tb-doughnut-widget-settings",

4
application/src/main/data/json/system/widget_types/horizontal_doughnut.json

@ -9,9 +9,9 @@
"sizeX": 4,
"sizeY": 2.5,
"resources": [],
"templateHtml": "<tb-doughnut-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-doughnut-widget>",
"templateHtml": "<tb-doughnut-widget \n [ctx]=\"ctx\">\n</tb-doughnut-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.doughnutWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.doughnutWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '380px',\n previewHeight: '220px',\n embedTitlePanel: true,\n horizontal: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind power', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar power', type: 'timeseries', color: '#FF4D5A' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.doughnutWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.doughnutWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '380px',\n previewHeight: '220px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n horizontal: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind power', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar power', type: 'timeseries', color: '#FF4D5A' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"settingsDirective": "tb-doughnut-widget-settings",

4
application/src/main/data/json/system/widget_types/image_map.json

@ -9,9 +9,9 @@
"sizeX": 8.5,
"sizeY": 6,
"resources": [],
"templateHtml": "<tb-map-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-map-widget>",
"templateHtml": "<tb-map-widget \n [ctx]=\"ctx\">\n</tb-map-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}\n",
"settingsForm": [],
"dataKeySettingsForm": [],
"settingsDirective": "tb-map-widget-settings",

4
application/src/main/data/json/system/widget_types/line_chart.json

@ -9,9 +9,9 @@
"sizeX": 8,
"sizeY": 5,
"resources": [],
"templateHtml": "<tb-time-series-chart-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-time-series-chart-widget>",
"templateHtml": "<tb-time-series-chart-widget \n [ctx]=\"ctx\">\n</tb-time-series-chart-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'line',\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('line'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'line',\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('line'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n",
"settingsSchema": "{}",
"dataKeySettingsSchema": "{}",
"latestDataKeySettingsSchema": "{}",

4
application/src/main/data/json/system/widget_types/map.json

@ -9,9 +9,9 @@
"sizeX": 8.5,
"sizeY": 6,
"resources": [],
"templateHtml": "<tb-map-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-map-widget>",
"templateHtml": "<tb-map-widget \n [ctx]=\"ctx\">\n</tb-map-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}\n",
"settingsForm": [],
"dataKeySettingsForm": [],
"settingsDirective": "tb-map-widget-settings",

4
application/src/main/data/json/system/widget_types/pie.json

@ -9,9 +9,9 @@
"sizeX": 5,
"sizeY": 4,
"resources": [],
"templateHtml": "<tb-pie-chart-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-pie-chart-widget>",
"templateHtml": "<tb-pie-chart-widget \n [ctx]=\"ctx\">\n</tb-pie-chart-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.pieChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.pieChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\nself.actionSources = function() {\n return {\n 'sliceClick': {\n name: 'widget-action.pie-slice-click',\n multiple: false\n }\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.pieChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.pieChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\nself.actionSources = function() {\n return {\n 'sliceClick': {\n name: 'widget-action.pie-slice-click',\n multiple: false\n }\n };\n}\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"settingsDirective": "tb-pie-chart-widget-settings",

4
application/src/main/data/json/system/widget_types/point_chart.json

@ -9,9 +9,9 @@
"sizeX": 8,
"sizeY": 5,
"resources": [],
"templateHtml": "<tb-time-series-chart-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-time-series-chart-widget>",
"templateHtml": "<tb-time-series-chart-widget \n [ctx]=\"ctx\">\n</tb-time-series-chart-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'point',\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('point'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'point',\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('point'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n",
"settingsSchema": "{}",
"dataKeySettingsSchema": "{}",
"latestDataKeySettingsSchema": "{}",

4
application/src/main/data/json/system/widget_types/polar_area.json

@ -9,9 +9,9 @@
"sizeX": 5,
"sizeY": 4,
"resources": [],
"templateHtml": "<tb-polar-area-chart-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-polar-area-chart-widget>",
"templateHtml": "<tb-polar-area-chart-widget \n [ctx]=\"ctx\">\n</tb-polar-area-chart-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.polarAreaChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.polarAreaChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.polarAreaChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.polarAreaChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"settingsDirective": "tb-polar-area-chart-widget-settings",

4
application/src/main/data/json/system/widget_types/radar.json

@ -9,9 +9,9 @@
"sizeX": 5,
"sizeY": 4,
"resources": [],
"templateHtml": "<tb-radar-chart-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-radar-chart-widget>",
"templateHtml": "<tb-radar-chart-widget \n [ctx]=\"ctx\">\n</tb-radar-chart-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.radarChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.radarChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.radarChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.radarChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n",
"settingsSchema": "",
"dataKeySettingsSchema": "",
"settingsDirective": "tb-radar-chart-widget-settings",

4
application/src/main/data/json/system/widget_types/range_chart.json

@ -9,9 +9,9 @@
"sizeX": 8,
"sizeY": 5,
"resources": [],
"templateHtml": "<tb-range-chart-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-range-chart-widget>",
"templateHtml": "<tb-range-chart-widget \n [ctx]=\"ctx\">\n</tb-range-chart-widget>",
"templateCss": ".legend {\n font-size: 13px;\n line-height: 10px;\n}\n\n.legend table { \n border-spacing: 0px;\n border-collapse: separate;\n}\n\n.mouse-events .flot-overlay {\n cursor: crosshair; \n}\n\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.rangeChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.rangeChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries' }];\n }\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.rangeChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.rangeChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries' }];\n }\n };\n}\n",
"settingsSchema": "{}",
"dataKeySettingsSchema": "{}",
"latestDataKeySettingsSchema": "{}",

4
application/src/main/data/json/system/widget_types/route_map.json

@ -9,9 +9,9 @@
"sizeX": 8.5,
"sizeY": 6,
"resources": [],
"templateHtml": "<tb-map-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-map-widget>",
"templateHtml": "<tb-map-widget \n [ctx]=\"ctx\">\n</tb-map-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n trip: true,\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n trip: true,\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}",
"settingsForm": [],
"dataKeySettingsForm": [],
"latestDataKeySettingsForm": [],

4
application/src/main/data/json/system/widget_types/state_chart.json

@ -9,9 +9,9 @@
"sizeX": 8,
"sizeY": 5,
"resources": [],
"templateHtml": "<tb-time-series-chart-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-time-series-chart-widget>",
"templateHtml": "<tb-time-series-chart-widget \n [ctx]=\"ctx\">\n</tb-time-series-chart-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n stateData: true,\n chartType: 'state',\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('state'),\n defaultDataKeysFunction: function() {\n return [{ name: 'state', label: 'State', type: 'timeseries', units: '', decimals: 0 }];\n }\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n stateData: true,\n chartType: 'state',\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('state'),\n defaultDataKeysFunction: function() {\n return [{ name: 'state', label: 'State', type: 'timeseries', units: '', decimals: 0 }];\n }\n };\n}\n",
"settingsSchema": "{}",
"dataKeySettingsSchema": "{}",
"latestDataKeySettingsSchema": "{}",

4
application/src/main/data/json/system/widget_types/time_series_chart.json

@ -9,9 +9,9 @@
"sizeX": 8,
"sizeY": 5,
"resources": [],
"templateHtml": "<tb-time-series-chart-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-time-series-chart-widget>",
"templateHtml": "<tb-time-series-chart-widget \n [ctx]=\"ctx\">\n</tb-time-series-chart-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings(),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings(),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n",
"settingsSchema": "{}",
"dataKeySettingsSchema": "{}",
"latestDataKeySettingsSchema": "{}",

4
application/src/main/data/json/system/widget_types/trip_map.json

@ -9,9 +9,9 @@
"sizeX": 8.5,
"sizeY": 6,
"resources": [],
"templateHtml": "<tb-map-widget \n [ctx]=\"ctx\"\n [widgetTitlePanel]=\"widgetTitlePanel\">\n</tb-map-widget>",
"templateHtml": "<tb-map-widget \n [ctx]=\"ctx\">\n</tb-map-widget>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n trip: true,\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n trip: true,\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}",
"settingsForm": [],
"dataKeySettingsForm": [],
"latestDataKeySettingsForm": [],

2
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java

@ -422,7 +422,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
private Map<String, ArgumentEntry> mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, List<String> removedTelemetryKeys) {
Map<String, Argument> deletedArguments = ctx.getArguments().entrySet().stream()
.filter(entry -> removedTelemetryKeys.contains(entry.getKey()))
.filter(entry -> removedTelemetryKeys.contains(entry.getValue().getRefEntityKey().getKey()))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
Map<String, ArgumentEntry> fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments);

5
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java

@ -97,7 +97,10 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta
.scheduler(eventConsumer.getScheduler())
.taskExecutor(eventConsumer.getTaskExecutor())
.build();
super.stateService = new KafkaQueueStateService<>(eventConsumer, stateConsumer);
super.stateService = KafkaQueueStateService.<TbProtoQueueMsg<ToCalculatedFieldMsg>, TbProtoQueueMsg<CalculatedFieldStateProto>>builder()
.eventConsumer(eventConsumer)
.stateConsumer(stateConsumer)
.build();
this.stateProducer = (TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>>) queueFactory.createCalculatedFieldStateProducer();
}

12
application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java

@ -59,7 +59,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.discovery.HashPartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.environment.DistributedLock;
import org.thingsboard.server.queue.environment.DistributedLockService;
import org.thingsboard.server.queue.provider.EdqsClientQueueFactory;
@ -96,7 +95,7 @@ public class DefaultEdqsService implements EdqsService {
private void init() {
executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass());
eventsProducer = EdqsProducer.builder()
.producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS))
.producer(queueFactory.createEdqsEventsProducer())
.partitionService(edqsPartitionService)
.build();
syncLock = distributedLockService.getLock("edqs_sync");
@ -148,9 +147,12 @@ public class DefaultEdqsService implements EdqsService {
syncLock.lock();
try {
EdqsSyncState syncState = getSyncState();
if (syncState != null && syncState.getStatus() == EdqsSyncStatus.FINISHED) {
log.info("EDQS sync is already finished");
return;
if (syncState != null) {
EdqsSyncStatus status = syncState.getStatus();
if (status == EdqsSyncStatus.FINISHED || status == EdqsSyncStatus.FAILED) {
log.info("EDQS sync is already " + status + ", ignoring the msg");
return;
}
}
saveSyncState(EdqsSyncStatus.STARTED);

3
application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java

@ -19,7 +19,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.queue.edqs.EdqsConfig;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaSettings;
@ -37,7 +36,7 @@ public class KafkaEdqsSyncService extends EdqsSyncService {
TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap());
this.syncNeeded = kafkaAdmin.areAllTopicsEmpty(IntStream.range(0, edqsConfig.getPartitions())
.mapToObj(partition -> TopicPartitionInfo.builder()
.topic(EdqsQueue.EVENTS.getTopic())
.topic(edqsConfig.getEventsTopic())
.partition(partition)
.build().getFullTopicName())
.collect(Collectors.toSet()));

18
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java

@ -54,7 +54,7 @@ import org.thingsboard.server.service.cf.CalculatedFieldCache;
import org.thingsboard.server.service.cf.CalculatedFieldStateService;
import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.queue.processing.AbstractConsumerPartitionedService;
import org.thingsboard.server.service.queue.processing.AbstractPartitionBasedConsumerService;
import org.thingsboard.server.service.queue.processing.IdMsgPair;
import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService;
@ -71,7 +71,7 @@ import java.util.stream.Collectors;
@Service
@TbRuleEngineComponent
@Slf4j
public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPartitionedService<ToCalculatedFieldNotificationMsg> implements TbCalculatedFieldConsumerService {
public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBasedConsumerService<ToCalculatedFieldNotificationMsg> implements TbCalculatedFieldConsumerService {
@Value("${queue.calculated_fields.poll_interval:25}")
private long pollInterval;
@ -99,7 +99,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar
}
@Override
protected void doAfterStartUp() {
protected void onStartUp() {
var queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME);
PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer = PartitionedQueueConsumerManager.<TbProtoQueueMsg<ToCalculatedFieldMsg>>create()
.queueKey(queueKey)
@ -126,7 +126,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar
}
@Override
protected void processPartitionChangeEvent(PartitionChangeEvent event) {
protected void onPartitionChangeEvent(PartitionChangeEvent event) {
try {
event.getNewPartitions().forEach((queueKey, partitions) -> {
if (queueKey.getQueueName().equals(DataConstants.CF_QUEUE_NAME)) {
@ -143,11 +143,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar
}
}
@Override
protected String getPrefix() {
return "tb-cf";
}
private void processMsgs(List<TbProtoQueueMsg<ToCalculatedFieldMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCalculatedFieldMsg>> consumer, QueueConfig config) throws Exception {
List<IdMsgPair<ToCalculatedFieldMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList();
ConcurrentMap<UUID, TbProtoQueueMsg<ToCalculatedFieldMsg>> pendingMap = orderedMsgList.stream().collect(
@ -195,6 +190,11 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar
return ServiceType.TB_RULE_ENGINE;
}
@Override
protected String getPrefix() {
return "tb-cf";
}
@Override
protected long getNotificationPollDuration() {
return pollInterval;

18
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -49,7 +49,7 @@ import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.cf.CalculatedFieldCache;
import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.queue.processing.AbstractConsumerPartitionedService;
import org.thingsboard.server.service.queue.processing.AbstractPartitionBasedConsumerService;
import org.thingsboard.server.service.queue.ruleengine.TbRuleEngineConsumerContext;
import org.thingsboard.server.service.queue.ruleengine.TbRuleEngineQueueConsumerManager;
import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
@ -66,7 +66,7 @@ import java.util.stream.Collectors;
@Service
@TbRuleEngineComponent
@Slf4j
public class DefaultTbRuleEngineConsumerService extends AbstractConsumerPartitionedService<ToRuleEngineNotificationMsg> implements TbRuleEngineConsumerService {
public class DefaultTbRuleEngineConsumerService extends AbstractPartitionBasedConsumerService<ToRuleEngineNotificationMsg> implements TbRuleEngineConsumerService {
private final TbRuleEngineConsumerContext ctx;
private final QueueService queueService;
@ -93,7 +93,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerPartitio
}
@Override
protected void doAfterStartUp() {
protected void onStartUp() {
List<Queue> queues = queueService.findAllQueues();
for (Queue configuration : queues) {
if (partitionService.isManagedByCurrentService(configuration.getTenantId())) {
@ -104,7 +104,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerPartitio
}
@Override
protected void processPartitionChangeEvent(PartitionChangeEvent event) {
protected void onPartitionChangeEvent(PartitionChangeEvent event) {
event.getNewPartitions().forEach((queueKey, partitions) -> {
if (DataConstants.CF_QUEUE_NAME.equals(queueKey.getQueueName()) || DataConstants.CF_STATES_QUEUE_NAME.equals(queueKey.getQueueName())) {
return;
@ -136,11 +136,6 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerPartitio
});
}
@Override
protected String getPrefix() {
return "tb-rule-engine";
}
@Override
protected void stopConsumers() {
super.stopConsumers();
@ -153,6 +148,11 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerPartitio
return ServiceType.TB_RULE_ENGINE;
}
@Override
protected String getPrefix() {
return "tb-rule-engine";
}
@Override
protected long getNotificationPollDuration() {
return ctx.getPollDuration();

43
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerPartitionedService.java → application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractPartitionBasedConsumerService.java

@ -31,24 +31,22 @@ import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsServ
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
public abstract class AbstractConsumerPartitionedService<N extends com.google.protobuf.GeneratedMessageV3> extends AbstractConsumerService<N> {
public abstract class AbstractPartitionBasedConsumerService<N extends com.google.protobuf.GeneratedMessageV3> extends AbstractConsumerService<N> {
private final Lock startupLock;
private volatile boolean consumersInitialized;
private final Lock startupLock = new ReentrantLock();
private volatile boolean started = false;
private PartitionChangeEvent lastPartitionChangeEvent;
public AbstractConsumerPartitionedService(ActorSystemContext actorContext,
TbTenantProfileCache tenantProfileCache,
TbDeviceProfileCache deviceProfileCache,
TbAssetProfileCache assetProfileCache,
CalculatedFieldCache calculatedFieldCache,
TbApiUsageStateService apiUsageStateService,
PartitionService partitionService,
ApplicationEventPublisher eventPublisher,
JwtSettingsService jwtSettingsService) {
public AbstractPartitionBasedConsumerService(ActorSystemContext actorContext,
TbTenantProfileCache tenantProfileCache,
TbDeviceProfileCache deviceProfileCache,
TbAssetProfileCache assetProfileCache,
CalculatedFieldCache calculatedFieldCache,
TbApiUsageStateService apiUsageStateService,
PartitionService partitionService,
ApplicationEventPublisher eventPublisher,
JwtSettingsService jwtSettingsService) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService);
this.startupLock = new ReentrantLock();
this.consumersInitialized = false;
}
@PostConstruct
@ -57,13 +55,14 @@ public abstract class AbstractConsumerPartitionedService<N extends com.google.pr
}
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
@Override
public void afterStartUp() {
super.afterStartUp();
doAfterStartUp();
onStartUp();
startupLock.lock();
try {
processPartitionChangeEvent(lastPartitionChangeEvent);
consumersInitialized = true;
onPartitionChangeEvent(lastPartitionChangeEvent);
started = true;
} finally {
startupLock.unlock();
}
@ -71,10 +70,10 @@ public abstract class AbstractConsumerPartitionedService<N extends com.google.pr
@Override
protected void onTbApplicationEvent(PartitionChangeEvent event) {
if (!consumersInitialized) {
if (!started) {
startupLock.lock();
try {
if (!consumersInitialized) {
if (!started) {
lastPartitionChangeEvent = event;
return;
}
@ -82,12 +81,12 @@ public abstract class AbstractConsumerPartitionedService<N extends com.google.pr
startupLock.unlock();
}
}
processPartitionChangeEvent(event);
onPartitionChangeEvent(event);
}
protected abstract void doAfterStartUp();
protected abstract void onStartUp();
protected abstract void processPartitionChangeEvent(PartitionChangeEvent event);
protected abstract void onPartitionChangeEvent(PartitionChangeEvent event);
protected abstract String getPrefix();

6
application/src/main/resources/thingsboard.yml

@ -1646,7 +1646,7 @@ queue:
# Kafka properties for Calculated Field State topics
calculated-field-state: "${TB_QUEUE_KAFKA_CF_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:104857600000;partitions:1;min.insync.replicas:1;cleanup.policy:compact}"
# Kafka properties for EDQS events topics
edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:1;min.insync.replicas:1}"
edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:86400000;segment.bytes:52428800;retention.bytes:-1;partitions:1;min.insync.replicas:1}"
# Kafka properties for EDQS requests topic (default: 3 minutes retention)
edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
# Kafka properties for EDQS state topic (infinite retention, compaction)
@ -1748,6 +1748,10 @@ queue:
partitions: "${TB_EDQS_PARTITIONS:12}"
# EDQS partitioning strategy: tenant (partition is resolved by tenant id) or none (no specific strategy, resolving by message key)
partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}"
# EDQS events topic
events_topic: "${TB_EDQS_EVENTS_TOPIC:edqs.events}"
# EDQS state topic
state_topic: "${TB_EDQS_STATE_TOPIC:edqs.state}"
# EDQS requests topic
requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}"
# EDQS responses topic

3
application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java

@ -17,8 +17,6 @@ package org.thingsboard.server.controller;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import org.awaitility.Awaitility;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -73,7 +71,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;

2
application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java

@ -110,7 +110,7 @@ public class EdqsEntityServiceTest extends EntityServiceTest {
@Override
protected List<EntityData> findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List<String> expectedTelemetries) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries),
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> loadAllData(query, expectedTelemetries.size()),
loadedEntities -> loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList().containsAll(expectedTelemetries));
}

19
application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java

@ -1698,6 +1698,19 @@ public class EntityServiceTest extends AbstractControllerTest {
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
findByQueryAndCheckTelemetry(query, EntityKeyType.TIME_SERIES, "temperature", deviceHighTemperatures);
// change sort order to sort by temperature
temperatures.sort(Comparator.naturalOrder());
List<String> expectedSortedList = temperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList());
EntityDataSortOrder sortByTempOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.TIME_SERIES, "temperature"), EntityDataSortOrder.Direction.ASC);
EntityDataPageLink sortByTempPageLink = new EntityDataPageLink(10, 0, null, sortByTempOrder);
EntityDataQuery querySortByTemp = new EntityDataQuery(filter, sortByTempPageLink, entityFields, latestValues, null);
List<EntityData> loadedEntities = loadAllData(querySortByTemp, deviceTemperatures.size());
List<String> entitiesTelemetry = loadedEntities.stream().map(entityData -> entityData.getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).toList();
assertThat(entitiesTelemetry).containsExactlyElementsOf(expectedSortedList);
deviceService.deleteDevicesByTenantId(tenantId);
}
@ -2377,14 +2390,14 @@ public class EntityServiceTest extends AbstractControllerTest {
}
protected List<EntityData> findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List<String> expectedTelemetry) {
List<EntityData> loadedEntities = findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetry);
List<EntityData> loadedEntities = loadAllData(query, expectedTelemetry.size());
List<String> entitiesTelemetry = loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList();
assertThat(entitiesTelemetry).containsExactlyInAnyOrderElementsOf(expectedTelemetry);
return loadedEntities;
}
protected List<EntityData> findEntitiesTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List<String> expectedTelemetries) {
PageData<EntityData> data = findByQueryAndCheck(query, expectedTelemetries.size());
protected List<EntityData> loadAllData(EntityDataQuery query, int expectedSize) {
PageData<EntityData> data = findByQueryAndCheck(query, expectedSize);
List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();

2
common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java

@ -17,7 +17,7 @@ package org.thingsboard.server.common.data.edqs;
import org.thingsboard.server.common.data.kv.DataType;
public interface DataPoint {
public interface DataPoint extends Comparable<DataPoint> {
String NOT_SUPPORTED = "Not supported!";

6
common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java

@ -50,10 +50,10 @@ public class CustomerData extends BaseEntityData<CustomerFields> {
entitiesById.computeIfAbsent(ed.getEntityType(), et -> new ConcurrentHashMap<>()).put(ed.getId(), ed);
}
public boolean remove(EntityData<?> ed) {
var map = entitiesById.get(ed.getEntityType());
public boolean remove(EntityType entityType, UUID entityId) {
var map = entitiesById.get(entityType);
if (map != null) {
return map.remove(ed.getId()) != null;
return map.remove(entityId) != null;
} else {
return false;
}

6
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java

@ -17,6 +17,7 @@ package org.thingsboard.server.edqs.data.dp;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.thingsboard.server.common.data.edqs.DataPoint;
@RequiredArgsConstructor
@ -54,4 +55,9 @@ public abstract class AbstractDataPoint implements DataPoint {
return valueToString();
}
@Override
public int compareTo(DataPoint dataPoint) {
return StringUtils.compareIgnoreCase(valueToString(), dataPoint.valueToString());
}
}

5
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java

@ -16,6 +16,7 @@
package org.thingsboard.server.edqs.data.dp;
import lombok.Getter;
import org.thingsboard.server.common.data.edqs.DataPoint;
import org.thingsboard.server.common.data.kv.DataType;
public class BoolDataPoint extends AbstractDataPoint {
@ -43,4 +44,8 @@ public class BoolDataPoint extends AbstractDataPoint {
return Boolean.toString(value);
}
@Override
public int compareTo(DataPoint dataPoint) {
return Boolean.compare(value, dataPoint.getBool());
}
}

5
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java

@ -16,6 +16,7 @@
package org.thingsboard.server.edqs.data.dp;
import lombok.Getter;
import org.thingsboard.server.common.data.edqs.DataPoint;
import org.thingsboard.server.common.data.kv.DataType;
public class DoubleDataPoint extends AbstractDataPoint {
@ -43,4 +44,8 @@ public class DoubleDataPoint extends AbstractDataPoint {
return Double.toString(value);
}
@Override
public int compareTo(DataPoint dataPoint) {
return Double.compare(value, dataPoint.getDouble());
}
}

6
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java

@ -16,6 +16,7 @@
package org.thingsboard.server.edqs.data.dp;
import lombok.Getter;
import org.thingsboard.server.common.data.edqs.DataPoint;
import org.thingsboard.server.common.data.kv.DataType;
public class LongDataPoint extends AbstractDataPoint {
@ -47,4 +48,9 @@ public class LongDataPoint extends AbstractDataPoint {
public String valueToString() {
return Long.toString(value);
}
@Override
public int compareTo(DataPoint dataPoint) {
return Long.compare(value, dataPoint.getLong());
}
}

15
common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java

@ -62,7 +62,6 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.thingsboard.server.queue.edqs.EdqsComponent;
import org.thingsboard.server.queue.edqs.EdqsConfig;
import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.edqs.EdqsQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp;
@ -123,8 +122,8 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
};
eventConsumer = PartitionedQueueConsumerManager.<TbProtoQueueMsg<ToEdqsMsg>>create()
.queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.EVENTS.getTopic()))
.topic(EdqsQueue.EVENTS.getTopic())
.queueKey(new QueueKey(ServiceType.EDQS, config.getEventsTopic()))
.topic(config.getEventsTopic())
.pollInterval(config.getPollInterval())
.msgPackProcessor((msgs, consumer, config) -> {
for (TbProtoQueueMsg<ToEdqsMsg> queueMsg : msgs) {
@ -133,14 +132,14 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
}
try {
ToEdqsMsg msg = queueMsg.getValue();
process(msg, EdqsQueue.EVENTS);
process(msg, true);
} catch (Exception t) {
log.error("Failed to process message: {}", queueMsg, t);
}
}
consumer.commit();
})
.consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS))
.consumerCreator((config, partitionId) -> queueFactory.createEdqsEventsConsumer())
.queueAdmin(queueFactory.getEdqsQueueAdmin())
.consumerExecutor(consumersExecutor)
.taskExecutor(taskExecutor)
@ -165,7 +164,7 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
try {
Set<TopicPartitionInfo> newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS));
stateService.process(withTopic(newPartitions, EdqsQueue.STATE.getTopic()));
stateService.process(withTopic(newPartitions, config.getStateTopic()));
// eventsConsumer's partitions are updated by stateService
responseTemplate.subscribe(withTopic(newPartitions, config.getRequestsTopic())); // TODO: we subscribe to partitions before we are ready. implement consumer-per-partition version for request template
@ -235,7 +234,7 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
return response;
}
public void process(ToEdqsMsg edqsMsg, EdqsQueue queue) {
public void process(ToEdqsMsg edqsMsg, boolean backup) {
log.trace("Processing message: {}", edqsMsg);
if (edqsMsg.hasEventMsg()) {
EdqsEventMsg eventMsg = edqsMsg.getEventMsg();
@ -252,7 +251,7 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
} else if (!ObjectType.unversionedTypes.contains(objectType)) {
log.warn("[{}] {} {} doesn't have version", tenantId, objectType, key);
}
if (queue != EdqsQueue.STATE) {
if (backup) {
stateService.save(tenantId, objectType, key, eventType, edqsMsg);
}

3
common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java

@ -16,6 +16,7 @@
package org.thingsboard.server.edqs.query;
import lombok.Data;
import org.thingsboard.server.common.data.edqs.DataPoint;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.edqs.data.EntityData;
@ -26,7 +27,7 @@ import java.util.UUID;
public class SortableEntityData {
private final EntityData entityData;
private String sortValue;
private DataPoint sortValue;
public UUID getId(){
return entityData.getId();

3
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java

@ -16,6 +16,7 @@
package org.thingsboard.server.edqs.query.processor;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.edqs.DataPoint;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.edqs.data.EntityData;
@ -50,7 +51,7 @@ public abstract class AbstractQueryProcessor<T extends EntityFilter> implements
protected SortableEntityData toSortData(EntityData<?> ed) {
SortableEntityData sortData = new SortableEntityData(ed);
sortData.setSortValue(getSortValue(ed, sortKey));
sortData.setSortValue(getSortValue(ed, sortKey, ctx));
return sortData;
}

3
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java

@ -34,6 +34,7 @@ import java.util.List;
import java.util.Queue;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
public abstract class AbstractRelationQueryProcessor<T extends EntityFilter> extends AbstractQueryProcessor<T> {
@ -89,7 +90,7 @@ public abstract class AbstractRelationQueryProcessor<T extends EntityFilter> ext
private List<SortableEntityData> processTenantQuery(Set<EntityData<?>> entities) {
return entities.stream()
.map(this::toSortData)
.toList();
.collect(Collectors.toList());
}
private List<SortableEntityData> processCustomerQuery(Set<EntityData<?>> entities) {

30
common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java

@ -150,8 +150,9 @@ public class TenantRepo {
}
} else if (RelationTypeGroup.DASHBOARD.equals(entity.getTypeGroup())) {
if (EntityRelation.CONTAINS_TYPE.equals(entity.getType()) && entity.getFrom().getEntityType() == EntityType.CUSTOMER) {
((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(entity.getFrom().getId(), CustomerData::new))
.addOrUpdate(getEntityMap(EntityType.DASHBOARD).get(entity.getTo().getId()));
CustomerData customerData = (CustomerData) getOrCreate(entity.getFrom());
EntityData<?> dashboardData = getOrCreate(entity.getTo());
customerData.addOrUpdate(dashboardData);
}
}
} finally {
@ -170,8 +171,10 @@ public class TenantRepo {
}
} else if (RelationTypeGroup.DASHBOARD.equals(entityRelation.getTypeGroup())) {
if (EntityRelation.CONTAINS_TYPE.equals(entityRelation.getType()) && entityRelation.getFrom().getEntityType() == EntityType.CUSTOMER) {
((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(entityRelation.getFrom().getId(), CustomerData::new))
.remove(getEntityMap(EntityType.DASHBOARD).get(entityRelation.getTo().getId()));
CustomerData customerData = (CustomerData) get(entityRelation.getFrom());
if (customerData != null) {
customerData.remove(EntityType.DASHBOARD, entityRelation.getTo().getId());
}
}
}
}
@ -197,13 +200,13 @@ public class TenantRepo {
entityData.setCustomerId(newCustomerId);
if (entityIdMismatch(oldCustomerId, newCustomerId)) {
if (oldCustomerId != null) {
CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId);
CustomerData old = (CustomerData) get(EntityType.CUSTOMER, oldCustomerId);
if (old != null) {
old.remove(entityData);
old.remove(entityType, entityId);
}
}
if (newCustomerId != null) {
CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new);
CustomerData newData = (CustomerData) getOrCreate(EntityType.CUSTOMER, newCustomerId);
newData.addOrUpdate(entityData);
}
}
@ -223,11 +226,12 @@ public class TenantRepo {
getEntitySet(entityType).remove(removed);
}
edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED));
UUID customerId = removed.getCustomerId();
if (customerId != null) {
CustomerData customerData = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(customerId);
CustomerData customerData = (CustomerData) get(EntityType.CUSTOMER, customerId);
if (customerData != null) {
customerData.remove(removed);
customerData.remove(entityType, entityId);
}
}
}
@ -303,7 +307,11 @@ public class TenantRepo {
}
private EntityData<?> get(EntityId entityId) {
return getEntityMap(entityId.getEntityType()).get(entityId.getId());
return get(entityId.getEntityType(), entityId.getId());
}
private EntityData<?> get(EntityType entityType, UUID entityId) {
return getEntityMap(entityType).get(entityId);
}
private EntityData<?> constructEntityData(EntityType entityType, UUID id) {
@ -425,7 +433,7 @@ public class TenantRepo {
EntityType entityType = entityId.getEntityType();
return switch (entityType) {
case CUSTOMER, TENANT -> {
EntityFields fields = getEntityMap(entityType).get(entityId.getId()).getFields();
EntityFields fields = get(entityId).getFields();
yield fields != null ? fields.getName() : "";
}
default -> throw new RuntimeException("Unsupported entity type: " + entityType);

50
common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java

@ -36,12 +36,14 @@ import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.common.state.KafkaQueueStateService;
import org.thingsboard.server.queue.common.state.QueueStateService;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.edqs.EdqsConfig;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.edqs.EdqsQueueFactory;
import org.thingsboard.server.queue.edqs.KafkaEdqsComponent;
import org.thingsboard.server.queue.edqs.KafkaEdqsQueueFactory;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
@ -56,8 +58,7 @@ public class KafkaEdqsStateService implements EdqsStateService {
private final EdqsConfig config;
private final EdqsPartitionService partitionService;
private final EdqsQueueFactory queueFactory;
private final TopicService topicService;
private final KafkaEdqsQueueFactory queueFactory;
@Autowired @Lazy
private EdqsProcessor edqsProcessor;
@ -73,15 +74,16 @@ public class KafkaEdqsStateService implements EdqsStateService {
@Override
public void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>> eventConsumer) {
TbKafkaAdmin queueAdmin = queueFactory.getEdqsQueueAdmin();
stateConsumer = PartitionedQueueConsumerManager.<TbProtoQueueMsg<ToEdqsMsg>>create()
.queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.STATE.getTopic()))
.topic(EdqsQueue.STATE.getTopic())
.queueKey(new QueueKey(ServiceType.EDQS, config.getStateTopic()))
.topic(config.getStateTopic())
.pollInterval(config.getPollInterval())
.msgPackProcessor((msgs, consumer, config) -> {
for (TbProtoQueueMsg<ToEdqsMsg> queueMsg : msgs) {
try {
ToEdqsMsg msg = queueMsg.getValue();
edqsProcessor.process(msg, EdqsQueue.STATE);
edqsProcessor.process(msg, false);
if (stateReadCount.incrementAndGet() % 100000 == 0) {
log.info("[state] Processed {} msgs", stateReadCount.get());
}
@ -91,15 +93,15 @@ public class KafkaEdqsStateService implements EdqsStateService {
}
consumer.commit();
})
.consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.STATE))
.queueAdmin(queueFactory.getEdqsQueueAdmin())
.consumerCreator((config, partitionId) -> queueFactory.createEdqsStateConsumer())
.queueAdmin(queueAdmin)
.consumerExecutor(eventConsumer.getConsumerExecutor())
.taskExecutor(eventConsumer.getTaskExecutor())
.scheduler(eventConsumer.getScheduler())
.uncaughtErrorHandler(edqsProcessor.getErrorHandler())
.build();
queueStateService = new KafkaQueueStateService<>(eventConsumer, stateConsumer);
TbKafkaConsumerTemplate<TbProtoQueueMsg<ToEdqsMsg>> eventsToBackupKafkaConsumer = queueFactory.createEdqsEventsToBackupConsumer();
eventsToBackupConsumer = QueueConsumerManager.<TbProtoQueueMsg<ToEdqsMsg>>builder()
.name("edqs-events-to-backup-consumer")
.pollInterval(config.getPollInterval())
@ -137,15 +139,35 @@ public class KafkaEdqsStateService implements EdqsStateService {
}
consumer.commit();
})
.consumerCreator(() -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS, "events-to-backup-consumer-group")) // shared by all instances consumer group
.consumerCreator(() -> eventsToBackupKafkaConsumer)
.consumerExecutor(eventConsumer.getConsumerExecutor())
.threadPrefix("edqs-events-to-backup")
.build();
stateProducer = EdqsProducer.builder()
.producer(queueFactory.createEdqsMsgProducer(EdqsQueue.STATE))
.producer(queueFactory.createEdqsStateProducer())
.partitionService(partitionService)
.build();
queueStateService = KafkaQueueStateService.<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<ToEdqsMsg>>builder()
.eventConsumer(eventConsumer)
.stateConsumer(stateConsumer)
.eventsStartOffsetsProvider(() -> {
// taking start offsets for events topics from the events-to-backup consumer group,
// since eventConsumer doesn't use consumer group management and thus offset tracking
// (because we need to be able to consume the same topic-partition by multiple instances)
Map<String, Long> offsets = new HashMap<>();
try {
queueAdmin.getConsumerGroupOffsets(eventsToBackupKafkaConsumer.getGroupId())
.forEach((topicPartition, offsetAndMetadata) -> {
offsets.put(topicPartition.topic(), offsetAndMetadata.offset());
});
} catch (Exception e) {
log.error("Failed to get consumer group offsets for {}", eventsToBackupKafkaConsumer.getGroupId(), e);
}
return offsets;
})
.build();
}
@Override
@ -153,7 +175,7 @@ public class KafkaEdqsStateService implements EdqsStateService {
if (queueStateService.getPartitions().isEmpty()) {
Set<TopicPartitionInfo> allPartitions = IntStream.range(0, config.getPartitions())
.mapToObj(partition -> TopicPartitionInfo.builder()
.topic(EdqsQueue.EVENTS.getTopic())
.topic(config.getEventsTopic())
.partition(partition)
.build())
.collect(Collectors.toSet());

5
common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java

@ -29,7 +29,6 @@ import org.thingsboard.server.edqs.util.EdqsRocksDb;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent;
import java.util.Set;
@ -61,14 +60,14 @@ public class LocalEdqsStateService implements EdqsStateService {
try {
ToEdqsMsg edqsMsg = ToEdqsMsg.parseFrom(value);
log.trace("[{}] Restored msg from RocksDB: {}", key, edqsMsg);
processor.process(edqsMsg, EdqsQueue.STATE);
processor.process(edqsMsg, false);
} catch (Exception e) {
log.error("[{}] Failed to restore value", key, e);
}
});
log.info("Restore completed");
}
eventConsumer.update(withTopic(partitions, EdqsQueue.EVENTS.getTopic()));
eventConsumer.update(withTopic(partitions, eventConsumer.getTopic()));
this.partitions = partitions;
}

21
common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java

@ -67,10 +67,10 @@ import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.Co
@Slf4j
public class RepositoryUtils {
public static final Comparator<SortableEntityData> SORT_ASC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse(""), String.CASE_INSENSITIVE_ORDER)
public static final Comparator<SortableEntityData> SORT_ASC = Comparator.comparing(SortableEntityData::getSortValue, Comparator.nullsFirst(Comparator.naturalOrder()))
.thenComparing(sp -> sp.getId().toString());
public static final Comparator<SortableEntityData> SORT_DESC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse(""), String.CASE_INSENSITIVE_ORDER)
public static final Comparator<SortableEntityData> SORT_DESC = Comparator.comparing(SortableEntityData::getSortValue, Comparator.nullsFirst(Comparator.naturalOrder()))
.thenComparing(sp -> sp.getId().toString()).reversed();
public static EntityType resolveEntityType(EntityFilter entityFilter) {
@ -348,24 +348,11 @@ public class RepositoryUtils {
}
}
public static String getSortValue(EntityData entity, DataKey sortKey) {
public static DataPoint getSortValue(EntityData entity, DataKey sortKey, QueryContext queryContext) {
if (sortKey == null) {
return null;
}
switch (sortKey.type()) {
case ENTITY_FIELD -> {
return entity.getField(sortKey.key());
}
case ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE, SERVER_ATTRIBUTE -> {
var dp = entity.getAttr(sortKey.keyId(), sortKey.type());
return dp != null ? dp.valueToString() : "";
}
case TIME_SERIES -> {
var dp = entity.getTs(sortKey.keyId());
return dp != null ? dp.valueToString() : "";
}
default -> throw new IllegalStateException("toSortKey is not implemented for type: " + sortKey.type());
}
return entity.getDataPoint(sortKey, queryContext);
}
public static boolean checkFilters(EdqsQuery query, EntityData entity) {

4
common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java

@ -90,9 +90,7 @@ public class TopicPartitionInfo {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
TopicPartitionInfo that = (TopicPartitionInfo) o;
return topic.equals(that.topic) &&
Objects.equals(tenantId, that.tenantId) &&
Objects.equals(partition, that.partition) &&
return Objects.equals(partition, that.partition) &&
fullTopicName.equals(that.fullTopicName);
}

12
common/message/src/test/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfoTest.java

@ -18,12 +18,16 @@ package org.thingsboard.server.common.msg.queue;
import org.junit.jupiter.api.Test;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.UUID;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.MatcherAssert.assertThat;
public class TopicPartitionInfoTest {
private final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID());
@Test
public void givenTopicPartitionInfo_whenEquals_thenTrue() {
@ -52,13 +56,13 @@ public class TopicPartitionInfoTest {
assertThat(TopicPartitionInfo.builder()
.topic("tb_core")
.tenantId(TenantId.SYS_TENANT_ID)
.tenantId(tenantId)
.partition(4)
.myPartition(true) //will ignored on equals
.build()
, is(TopicPartitionInfo.builder()
.topic("tb_core")
.tenantId(TenantId.SYS_TENANT_ID)
.tenantId(tenantId)
.partition(4)
.myPartition(true) //will ignored on equals
.build()));
@ -109,7 +113,7 @@ public class TopicPartitionInfoTest {
assertThat(TopicPartitionInfo.builder()
.topic("tb_core")
.tenantId(TenantId.SYS_TENANT_ID)
.tenantId(tenantId)
.partition(4)
.myPartition(true) //will ignored on equals
.build()
@ -117,7 +121,7 @@ public class TopicPartitionInfoTest {
assertThat(TopicPartitionInfo.builder()
.topic("tb_core")
.tenantId(TenantId.SYS_TENANT_ID)
.tenantId(tenantId)
.partition(4)
.myPartition(false) //will ignored on equals
.build()

14
common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java

@ -26,6 +26,7 @@ import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdateConfigTask;
import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdatePartitionsTask;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
import java.util.Collection;
import java.util.Collections;
@ -43,6 +44,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
@Slf4j
public class MainQueueConsumerManager<M extends TbQueueMsg, C extends QueueConfig> {
@ -296,7 +298,7 @@ public class MainQueueConsumerManager<M extends TbQueueMsg, C extends QueueConfi
log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, addedPartitions, removedPartitions);
removePartitions(removedPartitions);
addPartitions(addedPartitions, null);
addPartitions(addedPartitions, null, null);
}
protected void removePartitions(Set<TopicPartitionInfo> removedPartitions) {
@ -304,13 +306,19 @@ public class MainQueueConsumerManager<M extends TbQueueMsg, C extends QueueConfi
removedPartitions.forEach((tpi) -> Optional.ofNullable(consumers.remove(tpi)).ifPresent(TbQueueConsumerTask::awaitCompletion));
}
protected void addPartitions(Set<TopicPartitionInfo> partitions, Consumer<TopicPartitionInfo> onStop) {
protected void addPartitions(Set<TopicPartitionInfo> partitions, Consumer<TopicPartitionInfo> onStop, Function<String, Long> startOffsetProvider) {
partitions.forEach(tpi -> {
Integer partitionId = tpi.getPartition().orElse(-1);
String key = queueKey + "-" + partitionId;
Runnable callback = onStop != null ? () -> onStop.accept(tpi) : null;
TbQueueConsumerTask<M> consumer = new TbQueueConsumerTask<>(key, () -> consumerCreator.apply(config, partitionId), callback);
TbQueueConsumerTask<M> consumer = new TbQueueConsumerTask<>(key, () -> {
TbQueueConsumer<M> queueConsumer = consumerCreator.apply(config, partitionId);
if (startOffsetProvider != null && queueConsumer instanceof TbKafkaConsumerTemplate<M> kafkaConsumer) {
kafkaConsumer.setStartOffsetProvider(startOffsetProvider);
}
return queueConsumer;
}, callback);
consumers.put(tpi, consumer);
consumer.subscribe(Set.of(tpi));
launchConsumer(consumer);

9
common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java

@ -33,6 +33,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
@Slf4j
public class PartitionedQueueConsumerManager<M extends TbQueueMsg> extends MainQueueConsumerManager<M, QueueConfig> {
@ -57,7 +58,7 @@ public class PartitionedQueueConsumerManager<M extends TbQueueMsg> extends MainQ
protected void processTask(TbQueueConsumerManagerTask task) {
if (task instanceof AddPartitionsTask addPartitionsTask) {
log.info("[{}] Added partitions: {}", queueKey, addPartitionsTask.partitions());
consumerWrapper.addPartitions(addPartitionsTask.partitions(), addPartitionsTask.onStop());
consumerWrapper.addPartitions(addPartitionsTask.partitions(), addPartitionsTask.onStop(), addPartitionsTask.startOffsetProvider());
} else if (task instanceof RemovePartitionsTask removePartitionsTask) {
log.info("[{}] Removed partitions: {}", queueKey, removePartitionsTask.partitions());
consumerWrapper.removePartitions(removePartitionsTask.partitions());
@ -76,11 +77,11 @@ public class PartitionedQueueConsumerManager<M extends TbQueueMsg> extends MainQ
}
public void addPartitions(Set<TopicPartitionInfo> partitions) {
addPartitions(partitions, null);
addPartitions(partitions, null, null);
}
public void addPartitions(Set<TopicPartitionInfo> partitions, Consumer<TopicPartitionInfo> onStop) {
addTask(new AddPartitionsTask(partitions, onStop));
public void addPartitions(Set<TopicPartitionInfo> partitions, Consumer<TopicPartitionInfo> onStop, Function<String, Long> startOffsetProvider) {
addTask(new AddPartitionsTask(partitions, onStop, startOffsetProvider));
}
public void removePartitions(Set<TopicPartitionInfo> partitions) {

5
common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java

@ -20,6 +20,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
public interface TbQueueConsumerManagerTask {
@ -46,7 +47,9 @@ public interface TbQueueConsumerManagerTask {
}
}
record AddPartitionsTask(Set<TopicPartitionInfo> partitions, Consumer<TopicPartitionInfo> onStop) implements TbQueueConsumerManagerTask {
record AddPartitionsTask(Set<TopicPartitionInfo> partitions,
Consumer<TopicPartitionInfo> onStop,
Function<String, Long> startOffsetProvider) implements TbQueueConsumerManagerTask {
@Override
public QueueTaskType getType() {
return QueueTaskType.ADD_PARTITIONS;

16
common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java

@ -15,13 +15,16 @@
*/
package org.thingsboard.server.queue.common.state;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager;
import org.thingsboard.server.queue.discovery.QueueKey;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic;
@ -29,14 +32,21 @@ import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTop
public class KafkaQueueStateService<E extends TbQueueMsg, S extends TbQueueMsg> extends QueueStateService<E, S> {
private final PartitionedQueueConsumerManager<S> stateConsumer;
private final Supplier<Map<String, Long>> eventsStartOffsetsProvider;
public KafkaQueueStateService(PartitionedQueueConsumerManager<E> eventConsumer, PartitionedQueueConsumerManager<S> stateConsumer) {
@Builder
public KafkaQueueStateService(PartitionedQueueConsumerManager<E> eventConsumer,
PartitionedQueueConsumerManager<S> stateConsumer,
Supplier<Map<String, Long>> eventsStartOffsetsProvider) {
super(eventConsumer);
this.stateConsumer = stateConsumer;
this.eventsStartOffsetsProvider = eventsStartOffsetsProvider;
}
@Override
protected void addPartitions(QueueKey queueKey, Set<TopicPartitionInfo> partitions) {
Map<String, Long> eventsStartOffsets = eventsStartOffsetsProvider != null ? eventsStartOffsetsProvider.get() : null; // remembering the offsets before subscribing to states
Set<TopicPartitionInfo> statePartitions = withTopic(partitions, stateConsumer.getTopic());
partitionsInProgress.addAll(statePartitions);
stateConsumer.addPartitions(statePartitions, statePartition -> {
@ -51,12 +61,12 @@ public class KafkaQueueStateService<E extends TbQueueMsg, S extends TbQueueMsg>
TopicPartitionInfo eventPartition = statePartition.withTopic(eventConsumer.getTopic());
if (this.partitions.get(queueKey).contains(eventPartition)) {
eventConsumer.addPartitions(Set.of(eventPartition));
eventConsumer.addPartitions(Set.of(eventPartition), null, eventsStartOffsets != null ? eventsStartOffsets::get : null);
}
} finally {
readLock.unlock();
}
});
}, null);
}
@Override

9
common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java

@ -103,6 +103,9 @@ public class TopicService {
}
public String buildTopicName(String topic) {
if (topic == null) {
return null;
}
return prefix.isBlank() ? topic : prefix + "." + topic;
}
@ -113,9 +116,9 @@ public class TopicService {
public String buildConsumerGroupId(String servicePrefix, TenantId tenantId, String queueName, Integer partitionId) {
return this.buildTopicName(
servicePrefix + queueName
+ (tenantId.isSysTenantId() ? "" : ("-isolated-" + tenantId))
+ "-consumer"
+ suffix(partitionId));
+ (tenantId.isSysTenantId() ? "" : ("-isolated-" + tenantId))
+ "-consumer"
+ suffix(partitionId));
}
String suffix(Integer partitionId) {

2
common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java

@ -315,7 +315,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
ScheduledFuture<?> task = delayedTasks.remove(serviceId);
if (task != null) {
if (task.cancel(false)) {
log.debug("[{}] Recalculate partitions ignored. Service was restarted in time [{}].",
log.info("[{}] Recalculate partitions ignored. Service was restarted in time [{}].",
serviceId, serviceTypesList);
} else {
log.debug("[{}] Going to recalculate partitions. Service was not restarted in time [{}]!",

4
common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java

@ -30,6 +30,10 @@ public class EdqsConfig {
@Value("#{'${queue.edqs.partitioning_strategy:tenant}'.toUpperCase()}")
private EdqsPartitioningStrategy partitioningStrategy;
@Value("${queue.edqs.events_topic:edqs.events}")
private String eventsTopic;
@Value("${queue.edqs.state_topic:edqs.state}")
private String stateTopic;
@Value("${queue.edqs.requests_topic:edqs.requests}")
private String requestsTopic;
@Value("${queue.edqs.responses_topic:edqs.responses}")

36
common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java

@ -1,36 +0,0 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.edqs;
import lombok.Getter;
@Getter
public enum EdqsQueue {
EVENTS("edqs.events", false, false),
STATE("edqs.state", true, true);
private final String topic;
private final boolean readFromBeginning;
private final boolean stopWhenRead;
EdqsQueue(String topic, boolean readFromBeginning, boolean stopWhenRead) {
this.topic = topic;
this.readFromBeginning = readFromBeginning;
this.stopWhenRead = stopWhenRead;
}
}

8
common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java

@ -25,11 +25,13 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
public interface EdqsQueueFactory {
TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgConsumer(EdqsQueue queue);
TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsEventsConsumer();
TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgConsumer(EdqsQueue queue, String group);
TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsEventsToBackupConsumer();
TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgProducer(EdqsQueue queue);
TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsStateConsumer();
TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsStateProducer();
TbQueueResponseTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> createEdqsResponseTemplate();

23
common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java

@ -43,24 +43,23 @@ public class InMemoryEdqsQueueFactory implements EdqsQueueFactory {
private final TbQueueAdmin queueAdmin;
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgConsumer(EdqsQueue queue) {
if (queue == EdqsQueue.STATE) {
throw new UnsupportedOperationException();
}
return new InMemoryTbQueueConsumer<>(storage, queue.getTopic());
public TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsEventsConsumer() {
return new InMemoryTbQueueConsumer<>(storage, edqsConfig.getEventsTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgConsumer(EdqsQueue queue, String group) {
return createEdqsMsgConsumer(queue);
public TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsEventsToBackupConsumer() {
throw new UnsupportedOperationException();
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgProducer(EdqsQueue queue) {
if (queue == EdqsQueue.STATE) {
throw new UnsupportedOperationException();
}
return new InMemoryTbQueueProducer<>(storage, queue.getTopic());
public TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsStateConsumer() {
throw new UnsupportedOperationException();
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsStateProducer() {
throw new UnsupportedOperationException();
}
@Override

68
common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java

@ -19,11 +19,8 @@ import org.springframework.stereotype.Component;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.stats.StatsType;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueResponseTemplate;
import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate;
@ -71,55 +68,68 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory {
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgConsumer(EdqsQueue queue) {
String consumerGroup = "edqs-" + queue.name().toLowerCase() + "-consumer-group-" + serviceInfoProvider.getServiceId();
return createEdqsMsgConsumer(queue, consumerGroup);
public TbKafkaConsumerTemplate<TbProtoQueueMsg<ToEdqsMsg>> createEdqsEventsConsumer() {
return createEdqsMsgConsumer(edqsConfig.getEventsTopic(),
"edqs-events-" + consumerCounter.getAndIncrement() + "-consumer-" + serviceInfoProvider.getServiceId(),
null, // not using consumer group management, offsets from the edqs-events-to-backup-consumer-group are used (see KafkaEdqsStateService)
false, edqsEventsAdmin);
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgConsumer(EdqsQueue queue, String group) {
public TbKafkaConsumerTemplate<TbProtoQueueMsg<ToEdqsMsg>> createEdqsEventsToBackupConsumer() {
return createEdqsMsgConsumer(edqsConfig.getEventsTopic(),
"edqs-events-to-backup-consumer-" + serviceInfoProvider.getServiceId(),
"edqs-events-to-backup-consumer-group",
false, edqsEventsAdmin);
}
@Override
public TbKafkaConsumerTemplate<TbProtoQueueMsg<ToEdqsMsg>> createEdqsStateConsumer() {
return createEdqsMsgConsumer(edqsConfig.getStateTopic(),
"edqs-state-" + consumerCounter.getAndIncrement() + "-consumer-" + serviceInfoProvider.getServiceId(),
null, // not using consumer group management
true, edqsStateAdmin);
}
public TbKafkaConsumerTemplate<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgConsumer(String topic, String clientId, String group, boolean readFullAndStop, TbKafkaAdmin admin) {
return TbKafkaConsumerTemplate.<TbProtoQueueMsg<ToEdqsMsg>>builder()
.settings(kafkaSettings)
.topic(topicService.buildTopicName(queue.getTopic()))
.readFromBeginning(queue.isReadFromBeginning())
.stopWhenRead(queue.isStopWhenRead())
.clientId("edqs-" + queue.name().toLowerCase() + "-" + consumerCounter.getAndIncrement() + "-consumer-" + serviceInfoProvider.getServiceId())
.topic(topicService.buildTopicName(topic))
.readFromBeginning(readFullAndStop)
.stopWhenRead(readFullAndStop)
.clientId(clientId)
.groupId(topicService.buildTopicName(group))
.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders()))
.admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin)
.admin(admin)
.statsService(consumerStatsService)
.build();
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgProducer(EdqsQueue queue) {
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsStateProducer() {
return TbKafkaProducerTemplate.<TbProtoQueueMsg<ToEdqsMsg>>builder()
.clientId("edqs-" + queue.name().toLowerCase() + "-producer-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(queue.getTopic()))
.clientId("edqs-state-producer-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(edqsConfig.getStateTopic()))
.settings(kafkaSettings)
.admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin)
.admin(edqsStateAdmin)
.build();
}
@Override
public TbQueueResponseTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> createEdqsResponseTemplate() {
String requestsConsumerGroup = "edqs-requests-consumer-group-" + edqsConfig.getLabel();
var requestConsumer = TbKafkaConsumerTemplate.<TbProtoQueueMsg<TransportProtos.ToEdqsMsg>>builder()
.settings(kafkaSettings)
.topic(topicService.buildTopicName(edqsConfig.getRequestsTopic()))
.clientId("edqs-requests-consumer-" + serviceInfoProvider.getServiceId())
.groupId(topicService.buildTopicName(requestsConsumerGroup))
.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders()))
.admin(edqsRequestsAdmin)
.statsService(consumerStatsService);
var requestConsumer = createEdqsMsgConsumer(edqsConfig.getRequestsTopic(),
"edqs-requests-consumer-" + serviceInfoProvider.getServiceId(),
"edqs-requests-consumer-group",
false, edqsRequestsAdmin);
var responseProducer = TbKafkaProducerTemplate.<TbProtoQueueMsg<FromEdqsMsg>>builder()
.settings(kafkaSettings)
.clientId("edqs-response-producer-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(edqsConfig.getResponsesTopic()))
.admin(edqsRequestsAdmin);
.admin(edqsRequestsAdmin)
.build();
return DefaultTbQueueResponseTemplate.<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>>builder()
.requestTemplate(requestConsumer.build())
.responseTemplate(responseProducer.build())
.requestTemplate(requestConsumer)
.responseTemplate(responseProducer)
.maxPendingRequests(edqsConfig.getMaxPendingRequests())
.requestTimeout(edqsConfig.getMaxRequestTimeout())
.pollInterval(edqsConfig.getPollInterval())
@ -129,7 +139,7 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory {
}
@Override
public TbQueueAdmin getEdqsQueueAdmin() {
public TbKafkaAdmin getEdqsQueueAdmin() {
return edqsEventsAdmin;
}

14
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java

@ -15,6 +15,8 @@
*/
package org.thingsboard.server.queue.kafka;
import lombok.Getter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.admin.CreateTopicsResult;
import org.apache.kafka.clients.admin.ListOffsetsResult;
@ -45,6 +47,7 @@ public class TbKafkaAdmin implements TbQueueAdmin {
private final TbKafkaSettings settings;
private final Map<String, String> topicConfigs;
@Getter
private final int numPartitions;
private volatile Set<String> topics;
@ -157,8 +160,7 @@ public class TbKafkaAdmin implements TbQueueAdmin {
if (partitionId == null) {
return;
}
Map<TopicPartition, OffsetAndMetadata> oldOffsets =
settings.getAdminClient().listConsumerGroupOffsets(fatGroupId).partitionsToOffsetAndMetadata().get(10, TimeUnit.SECONDS);
Map<TopicPartition, OffsetAndMetadata> oldOffsets = getConsumerGroupOffsets(fatGroupId);
if (oldOffsets.isEmpty()) {
return;
}
@ -169,8 +171,7 @@ public class TbKafkaAdmin implements TbQueueAdmin {
continue;
}
var om = consumerOffset.getValue();
Map<TopicPartition, OffsetAndMetadata> newOffsets =
settings.getAdminClient().listConsumerGroupOffsets(newGroupId).partitionsToOffsetAndMetadata().get(10, TimeUnit.SECONDS);
Map<TopicPartition, OffsetAndMetadata> newOffsets = getConsumerGroupOffsets(newGroupId);
var existingOffset = newOffsets.get(tp);
if (existingOffset == null) {
@ -187,6 +188,11 @@ public class TbKafkaAdmin implements TbQueueAdmin {
}
}
@SneakyThrows
public Map<TopicPartition, OffsetAndMetadata> getConsumerGroupOffsets(String groupId) {
return settings.getAdminClient().listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata().get(10, TimeUnit.SECONDS);
}
public boolean isTopicEmpty(String topic) {
return areAllTopicsEmpty(Set.of(topic));
}

46
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java

@ -16,6 +16,8 @@
package org.thingsboard.server.queue.kafka;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
@ -39,7 +41,9 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
/**
* Created by ashvayka on 24.09.18.
@ -47,13 +51,16 @@ import java.util.stream.Collectors;
@Slf4j
public class TbKafkaConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQueueConsumerTemplate<ConsumerRecord<String, byte[]>, T> {
private final TbQueueAdmin admin;
private final TbKafkaAdmin admin;
private final KafkaConsumer<String, byte[]> consumer;
private final TbKafkaDecoder<T> decoder;
private final TbKafkaConsumerStatsService statsService;
@Getter
private final String groupId;
@Setter
private Function<String, Long> startOffsetProvider;
private final boolean readFromBeginning; // reset offset to beginning
private final boolean stopWhenRead; // stop consuming when reached end offset remembered on start
private int readCount;
@ -78,7 +85,7 @@ public class TbKafkaConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQue
statsService.registerClientGroup(groupId);
}
this.admin = admin;
this.admin = (TbKafkaAdmin) admin;
this.consumer = new KafkaConsumer<>(props);
this.decoder = decoder;
this.readFromBeginning = readFromBeginning;
@ -105,14 +112,19 @@ public class TbKafkaConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQue
List<String> toSubscribe = new ArrayList<>();
topics.forEach((topic, kafkaPartitions) -> {
if (kafkaPartitions == null) {
toSubscribe.add(topic);
} else {
List<TopicPartition> topicPartitions = kafkaPartitions.stream()
.map(partition -> new TopicPartition(topic, partition))
.toList();
consumer.assign(topicPartitions);
onPartitionsAssigned(topicPartitions);
if (groupId != null) {
toSubscribe.add(topic);
return;
} else { // if no consumer group management - manually assigning all topic partitions
kafkaPartitions = IntStream.range(0, admin.getNumPartitions()).boxed().toList();
}
}
List<TopicPartition> topicPartitions = kafkaPartitions.stream()
.map(partition -> new TopicPartition(topic, partition))
.toList();
consumer.assign(topicPartitions);
onPartitionsAssigned(topicPartitions);
});
if (!toSubscribe.isEmpty()) {
if (readFromBeginning || stopWhenRead) {
@ -179,9 +191,21 @@ public class TbKafkaConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQue
private void onPartitionsAssigned(Collection<TopicPartition> partitions) {
if (readFromBeginning) {
log.debug("Seeking to beginning for {}", partitions);
consumer.seekToBeginning(partitions);
} else if (startOffsetProvider != null) {
partitions.forEach(topicPartition -> {
Long offset = startOffsetProvider.apply(topicPartition.topic());
if (offset != null) {
log.debug("Seeking to offset {} for {}", offset, topicPartition);
consumer.seek(topicPartition, offset);
} else {
log.info("No start offset provided for {}", topicPartition);
}
});
}
if (stopWhenRead) {
log.debug("Getting end offsets for {}", partitions);
endOffsets = consumer.endOffsets(partitions).entrySet().stream()
.filter(entry -> entry.getValue() > 0)
.collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue));
@ -195,7 +219,9 @@ public class TbKafkaConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQue
@Override
protected void doCommit() {
consumer.commitSync();
if (groupId != null) {
consumer.commitSync();
}
}
@Override

3
common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java

@ -20,11 +20,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.edqs.EdqsQueue;
public interface EdqsClientQueueFactory {
TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgProducer(EdqsQueue queue);
TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsEventsProducer();
TbQueueRequestTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> createEdqsRequestTemplate();

5
common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java

@ -37,7 +37,6 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.edqs.EdqsConfig;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer;
import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer;
@ -239,8 +238,8 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgProducer(EdqsQueue queue) {
return new InMemoryTbQueueProducer<>(storage, queue.getTopic());
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsEventsProducer() {
return new InMemoryTbQueueProducer<>(storage, edqsConfig.getEventsTopic());
}
@Override

7
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java

@ -54,7 +54,6 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.edqs.EdqsConfig;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
@ -590,10 +589,10 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgProducer(EdqsQueue queue) {
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsEventsProducer() {
return TbKafkaProducerTemplate.<TbProtoQueueMsg<ToEdqsMsg>>builder()
.clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(queue.getTopic()))
.clientId("edqs-events-producer-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(edqsConfig.getEventsTopic()))
.settings(kafkaSettings)
.admin(edqsEventsAdmin)
.build();

7
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java

@ -52,7 +52,6 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.edqs.EdqsConfig;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
@ -480,10 +479,10 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgProducer(EdqsQueue queue) {
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsEventsProducer() {
return TbKafkaProducerTemplate.<TbProtoQueueMsg<ToEdqsMsg>>builder()
.clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(queue.getTopic()))
.clientId("edqs-events-producer-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(edqsConfig.getEventsTopic()))
.settings(kafkaSettings)
.admin(edqsEventsAdmin)
.build();

14
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java

@ -48,7 +48,7 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.edqs.EdqsConfig;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
@ -79,6 +79,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbQueueEdgeSettings edgeSettings;
private final TbQueueCalculatedFieldSettings calculatedFieldSettings;
private final EdqsConfig edqsConfig;
private final TbQueueAdmin coreAdmin;
private final TbKafkaAdmin ruleEngineAdmin;
@ -101,7 +102,9 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
TbQueueRemoteJsInvokeSettings jsInvokeSettings,
TbKafkaConsumerStatsService consumerStatsService,
TbQueueTransportNotificationSettings transportNotificationSettings,
TbQueueEdgeSettings edgeSettings, TbQueueCalculatedFieldSettings calculatedFieldSettings,
TbQueueEdgeSettings edgeSettings,
TbQueueCalculatedFieldSettings calculatedFieldSettings,
EdqsConfig edqsConfig,
TbKafkaTopicConfigs kafkaTopicConfigs) {
this.topicService = topicService;
this.kafkaSettings = kafkaSettings;
@ -113,6 +116,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
this.transportNotificationSettings = transportNotificationSettings;
this.edgeSettings = edgeSettings;
this.calculatedFieldSettings = calculatedFieldSettings;
this.edqsConfig = edqsConfig;
this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs());
this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs());
@ -385,10 +389,10 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsMsgProducer(EdqsQueue queue) {
public TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsEventsProducer() {
return TbKafkaProducerTemplate.<TbProtoQueueMsg<ToEdqsMsg>>builder()
.clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(queue.getTopic()))
.clientId("edqs-events-producer-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(edqsConfig.getEventsTopic()))
.settings(kafkaSettings)
.admin(edqsEventsAdmin)
.build();

6
edqs/src/main/resources/edqs.yml

@ -57,6 +57,10 @@ queue:
partitions: "${TB_EDQS_PARTITIONS:12}"
# EDQS partitioning strategy: tenant (partitions are resolved and distributed by tenant id) or none (partitions are resolved by message key; each instance has all the partitions)
partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}"
# EDQS events topic
events_topic: "${TB_EDQS_EVENTS_TOPIC:edqs.events}"
# EDQS state topic
state_topic: "${TB_EDQS_STATE_TOPIC:edqs.state}"
# EDQS requests topic
requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}"
# EDQS responses topic
@ -149,7 +153,7 @@ queue:
# value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds)
topic-properties:
# Kafka properties for EDQS events topics
edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:1;min.insync.replicas:1}"
edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:86400000;segment.bytes:52428800;retention.bytes:-1;partitions:1;min.insync.replicas:1}"
# Kafka properties for EDQS requests topic (default: 3 minutes retention)
edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
# Kafka properties for EDQS state topic (infinite retention, compaction)

18
pom.xml

@ -42,6 +42,7 @@
<jakarta.xml.bind-api.version>4.0.2</jakarta.xml.bind-api.version>
<javax.xml.bind-api.version>2.4.0-b180830.0359</javax.xml.bind-api.version>
<jaxb-runtime.version>4.0.5</jaxb-runtime.version>
<tomcat.version>10.1.39</tomcat.version> <!--Remove after update spring-boot to new version-->
<spring-boot.version>3.2.12</spring-boot.version>
<spring-data.version>3.2.12</spring-data.version>
<spring-data-redis.version>3.2.12</spring-data-redis.version>
@ -88,7 +89,7 @@
<lombok.version>1.18.32</lombok.version>
<paho.client.version>1.2.5</paho.client.version>
<paho.mqttv5.client.version>1.2.5</paho.mqttv5.client.version>
<netty.version>4.1.115.Final</netty.version>
<netty.version>4.1.119.Final</netty.version>
<netty-tcnative.version>2.0.65.Final</netty-tcnative.version>
<reactor-netty.version>1.1.18</reactor-netty.version>
<os-maven-plugin.version>1.7.1</os-maven-plugin.version>
@ -1147,6 +1148,21 @@
<artifactId>jaxb-runtime</artifactId>
<version>${jaxb-runtime.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<version>${tomcat.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-el</artifactId>
<version>${tomcat.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-websocket</artifactId>
<version>${tomcat.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>

9
ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.html

@ -17,7 +17,14 @@
-->
<div class="tb-bar-chart-panel" [style.padding]="padding" [style]="backgroundStyle$ | async">
<div class="tb-bar-chart-overlay" [style]="overlayStyle"></div>
<ng-container *ngTemplateOutlet="widgetTitlePanel"></ng-container>
@if (widgetComponent.dashboardWidget.showWidgetTitlePanel) {
<div class="tb-widget-title-row flex justify-between">
<ng-container *ngTemplateOutlet="widgetComponent.widgetTitlePanel"></ng-container>
<ng-container *ngTemplateOutlet="widgetComponent.widgetHeaderActionsPanel"></ng-container>
</div>
} @else {
<ng-container class="tb-widget-actions-absolute" *ngTemplateOutlet="widgetComponent.widgetHeaderActionsPanel; context: {absolute: true}"></ng-container>
}
<div class="tb-bar-chart-content" [class]="legendClass">
<div #chartShape class="tb-bar-chart-shape">
</div>

7
ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.scss

@ -104,4 +104,11 @@
}
}
}
.tb-widget-title-row .tb-widget-actions {
margin: 0;
}
.tb-widget-actions.tb-widget-actions-absolute {
z-index: 2;
}
}

7
ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts

@ -40,6 +40,7 @@ import {
} from '@home/components/widget/lib/chart/bar-chart-with-labels-widget.models';
import { TbTimeSeriesChart } from '@home/components/widget/lib/chart/time-series-chart';
import { DataKey } from '@shared/models/widget.models';
import { WidgetComponent } from '@home/components/widget/widget.component';
@Component({
selector: 'tb-bar-chart-with-labels-widget',
@ -57,9 +58,6 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft
@Input()
ctx: WidgetContext;
@Input()
widgetTitlePanel: TemplateRef<any>;
showLegend: boolean;
legendClass: string;
@ -73,7 +71,8 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft
private timeSeriesChart: TbTimeSeriesChart;
constructor(private imagePipe: ImagePipe,
constructor(public widgetComponent: WidgetComponent,
private imagePipe: ImagePipe,
private sanitizer: DomSanitizer,
private renderer: Renderer2,
private cd: ChangeDetectorRef) {

1
ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart-widget.component.html

@ -19,6 +19,5 @@
#latestChart
[ctx]="ctx"
[settings]="settings"
[widgetTitlePanel]="widgetTitlePanel"
[callbacks]="callbacks"
></tb-latest-chart>

9
ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart.component.html

@ -17,7 +17,14 @@
-->
<div class="tb-latest-chart-panel" [style.padding]="padding" [style]="backgroundStyle$ | async">
<div class="tb-latest-chart-overlay" [style]="overlayStyle"></div>
<ng-container *ngTemplateOutlet="widgetTitlePanel"></ng-container>
@if (widgetComponent.dashboardWidget.showWidgetTitlePanel) {
<div class="tb-widget-title-row flex justify-between">
<ng-container *ngTemplateOutlet="widgetComponent.widgetTitlePanel"></ng-container>
<ng-container *ngTemplateOutlet="widgetComponent.widgetHeaderActionsPanel"></ng-container>
</div>
} @else {
<ng-container class="tb-widget-actions-absolute" *ngTemplateOutlet="widgetComponent.widgetHeaderActionsPanel; context: {absolute: true}"></ng-container>
}
<div #chartContent class="tb-latest-chart-content" [class]="legendClass">
<div #chartShape class="tb-latest-chart-shape">
</div>

7
ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart.component.scss

@ -106,4 +106,11 @@
}
}
}
.tb-widget-title-row .tb-widget-actions {
margin: 0;
}
.tb-widget-actions.tb-widget-actions-absolute {
z-index: 2;
}
}

7
ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart.component.ts

@ -68,9 +68,6 @@ export class LatestChartComponent implements OnInit, OnDestroy, AfterViewInit {
@Input()
ctx: WidgetContext;
@Input()
widgetTitlePanel: TemplateRef<any>;
@Input()
callbacks: LatestChartComponentCallbacks;
@ -98,9 +95,9 @@ export class LatestChartComponent implements OnInit, OnDestroy, AfterViewInit {
private latestChart: TbLatestChart<LatestChartSettings>;
constructor(private imagePipe: ImagePipe,
constructor(public widgetComponent: WidgetComponent,
private imagePipe: ImagePipe,
private sanitizer: DomSanitizer,
private widgetComponent: WidgetComponent,
private renderer: Renderer2,
private translate: TranslateService,
private cd: ChangeDetectorRef) {

9
ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.html

@ -17,7 +17,14 @@
-->
<div class="tb-range-chart-panel" [style.padding]="padding" [style]="backgroundStyle$ | async">
<div class="tb-range-chart-overlay" [style]="overlayStyle"></div>
<ng-container *ngTemplateOutlet="widgetTitlePanel"></ng-container>
@if (widgetComponent.dashboardWidget.showWidgetTitlePanel) {
<div class="tb-widget-title-row flex justify-between">
<ng-container *ngTemplateOutlet="widgetComponent.widgetTitlePanel"></ng-container>
<ng-container *ngTemplateOutlet="widgetComponent.widgetHeaderActionsPanel"></ng-container>
</div>
} @else {
<ng-container class="tb-widget-actions-absolute" *ngTemplateOutlet="widgetComponent.widgetHeaderActionsPanel; context: {absolute: true}"></ng-container>
}
<div class="tb-range-chart-content" [class]="legendClass">
<div #chartShape class="tb-range-chart-shape">
</div>

6
ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.scss

@ -104,4 +104,10 @@
}
}
}
.tb-widget-title-row .tb-widget-actions {
margin: 0;
}
.tb-widget-actions.tb-widget-actions-absolute {
z-index: 2;
}
}

7
ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts

@ -48,6 +48,7 @@ import { Observable } from 'rxjs';
import { ImagePipe } from '@shared/pipe/image.pipe';
import { DomSanitizer } from '@angular/platform-browser';
import { TbTimeSeriesChart } from '@home/components/widget/lib/chart/time-series-chart';
import { WidgetComponent } from '@home/components/widget/widget.component';
@Component({
selector: 'tb-range-chart-widget',
@ -65,9 +66,6 @@ export class RangeChartWidgetComponent implements OnInit, OnDestroy, AfterViewIn
@Input()
ctx: WidgetContext;
@Input()
widgetTitlePanel: TemplateRef<any>;
showLegend: boolean;
legendClass: string;
@ -86,7 +84,8 @@ export class RangeChartWidgetComponent implements OnInit, OnDestroy, AfterViewIn
private timeSeriesChart: TbTimeSeriesChart;
constructor(private imagePipe: ImagePipe,
constructor(public widgetComponent: WidgetComponent,
private imagePipe: ImagePipe,
private sanitizer: DomSanitizer,
private renderer: Renderer2,
private cd: ChangeDetectorRef) {

11
ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.component.html

@ -17,9 +17,14 @@
-->
<div class="tb-time-series-chart-panel" [style.padding]="padding" [style]="backgroundStyle$ | async">
<div class="tb-time-series-chart-overlay" [style]="overlayStyle"></div>
<ng-container *ngIf="widgetComponent.dashboardWidget.showWidgetTitlePanel">
<ng-container *ngTemplateOutlet="widgetTitlePanel"></ng-container>
</ng-container>
@if (widgetComponent.dashboardWidget.showWidgetTitlePanel) {
<div class="tb-widget-title-row flex justify-between">
<ng-container *ngTemplateOutlet="widgetComponent.widgetTitlePanel"></ng-container>
<ng-container *ngTemplateOutlet="widgetComponent.widgetHeaderActionsPanel"></ng-container>
</div>
} @else {
<ng-container class="tb-widget-actions-absolute" *ngTemplateOutlet="widgetComponent.widgetHeaderActionsPanel; context: {absolute: true}"></ng-container>
}
<div class="tb-time-series-chart-content" [class]="legendClass">
<div #chartShape class="tb-time-series-chart-shape">
</div>

6
ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.component.scss

@ -156,4 +156,10 @@ $maxLegendHeight: 35%;
}
}
}
.tb-widget-title-row .tb-widget-actions {
margin: 0;
}
.tb-widget-actions.tb-widget-actions-absolute {
z-index: 2;
}
}

3
ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.component.ts

@ -61,9 +61,6 @@ export class TimeSeriesChartWidgetComponent implements OnInit, OnDestroy, AfterV
@Input()
ctx: WidgetContext;
@Input()
widgetTitlePanel: TemplateRef<any>;
horizontalLegendPosition = false;
showLegend: boolean;

2
ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts

@ -793,7 +793,7 @@ export class TbTimeSeriesChart {
}
} else {
if (!axis.option.name) {
axis.option.name = axis.settings.label;
axis.option.name = this.ctx.utilsService.customTranslation(axis.settings.label, axis.settings.label);
result.changed = true;
}
const nameGap = size;

7
ui-ngx/src/app/modules/home/components/widget/lib/maps/data-layer/circles-data-layer.ts

@ -22,7 +22,7 @@ import {
TbMapDatasource
} from '@shared/models/widget/maps/map.models';
import L from 'leaflet';
import { FormattedData } from '@shared/models/widget.models';
import { DataKey, FormattedData } from '@shared/models/widget.models';
import { TbShapesDataLayer } from '@home/components/widget/lib/maps/data-layer/shapes-data-layer';
import { TbMap } from '@home/components/widget/lib/maps/map';
import { Observable } from 'rxjs';
@ -213,9 +213,8 @@ export class TbCirclesDataLayer extends TbShapesDataLayer<CirclesDataLayerSettin
);
}
protected setupDatasource(datasource: TbMapDatasource): TbMapDatasource {
datasource.dataKeys.push(this.settings.circleKey);
return datasource;
protected getDataKeys(): DataKey[] {
return [this.settings.circleKey];
}
protected defaultBaseSettings(map: TbMap<any>): Partial<CirclesDataLayerSettings> {

45
ui-ngx/src/app/modules/home/components/widget/lib/maps/data-layer/map-data-layer.ts

@ -41,7 +41,7 @@ import L from 'leaflet';
import { CompiledTbFunction } from '@shared/models/js-function.models';
import { forkJoin, Observable, of } from 'rxjs';
import { map } from 'rxjs/operators';
import { DataKey, FormattedData } from '@shared/models/widget.models';
import { DataKey, DatasourceType, FormattedData } from '@shared/models/widget.models';
import { CustomTranslatePipe } from '@shared/pipe/custom-translate.pipe';
import { TbMap } from '@home/components/widget/lib/maps/map';
import { WidgetContext } from '@home/models/widget-component.models';
@ -170,7 +170,7 @@ export abstract class TbMapDataLayer<S extends MapDataLayerSettings = MapDataLay
protected settings: S;
protected datasource: TbMapDatasource;
protected dataSources: TbMapDatasource[];
protected mapDataId: string;
@ -202,13 +202,19 @@ export abstract class TbMapDataLayer<S extends MapDataLayerSettings = MapDataLay
}
public setup(): Observable<any> {
this.datasource = mapDataSourceSettingsToDatasource(this.settings);
this.datasource.dataKeys = this.settings.additionalDataKeys ? [...this.settings.additionalDataKeys] : [];
const colorRangeKeys = this.allColorSettings().filter(settings => settings.type === DataLayerColorType.range && settings.rangeKey)
.map(settings => settings.rangeKey);
this.datasource.dataKeys.push(...colorRangeKeys);
this.mapDataId = this.datasource.mapDataIds[0];
this.datasource = this.setupDatasource(this.datasource);
this.dataSources = [];
const datasource = mapDataSourceSettingsToDatasource(this.settings);
this.mapDataId = datasource.mapDataIds[0];
this.dataSources.push(datasource);
if (this.settings.additionalDataSources?.length && datasource.type !== DatasourceType.function) {
this.dataSources.push(...this.settings.additionalDataSources.map(ds => mapDataSourceSettingsToDatasource(ds, this.mapDataId)));
}
const dataKeys = this.calculateDataKeys();
const latestDataKeys = this.calculateLatestDataKeys();
this.dataSources.forEach(ds => ds.dataKeys.push(...dataKeys));
if (latestDataKeys?.length) {
this.dataSources.forEach(ds => ds.latestDataKeys.push(...latestDataKeys));
}
return forkJoin(
[
this.dataLayerLabelProcessor ? this.dataLayerLabelProcessor.setup() : of(null),
@ -241,8 +247,8 @@ export abstract class TbMapDataLayer<S extends MapDataLayerSettings = MapDataLay
return this.map.type();
}
public getDatasource(): TbMapDatasource {
return this.datasource;
public getDataSources(): TbMapDatasource[] {
return this.dataSources;
}
public getDataLayerContainer(): L.FeatureGroup {
@ -289,8 +295,21 @@ export abstract class TbMapDataLayer<S extends MapDataLayerSettings = MapDataLay
return L.featureGroup([], {snapIgnore: true});
}
protected setupDatasource(datasource: TbMapDatasource): TbMapDatasource {
return datasource;
protected calculateDataKeys(): DataKey[] {
const dataKeys = this.settings.additionalDataKeys ? [...this.settings.additionalDataKeys] : [];
const colorRangeKeys = this.allColorSettings().filter(settings => settings.type === DataLayerColorType.range && settings.rangeKey)
.map(settings => settings.rangeKey);
dataKeys.push(...colorRangeKeys);
dataKeys.push(...this.getDataKeys());
return dataKeys;
}
protected calculateLatestDataKeys(): DataKey[] {
return [];
}
protected getDataKeys(): DataKey[] {
return [];
}
protected allColorSettings(): DataLayerColorSettings[] {

7
ui-ngx/src/app/modules/home/components/widget/lib/maps/data-layer/markers-data-layer.ts

@ -37,7 +37,7 @@ import {
TbMapDatasource
} from '@shared/models/widget/maps/map.models';
import L, { FeatureGroup } from 'leaflet';
import { FormattedData } from '@shared/models/widget.models';
import { DataKey, FormattedData } from '@shared/models/widget.models';
import { forkJoin, Observable, of } from 'rxjs';
import { CompiledTbFunction } from '@shared/models/js-function.models';
import {
@ -628,9 +628,8 @@ export class TbMarkersDataLayer extends TbLatestMapDataLayer<MarkersDataLayerSet
}
}
protected setupDatasource(datasource: TbMapDatasource): TbMapDatasource {
datasource.dataKeys.push(this.settings.xKey, this.settings.yKey);
return datasource;
protected getDataKeys(): DataKey[] {
return [this.settings.xKey, this.settings.yKey];
}
protected allColorSettings(): DataLayerColorSettings[] {

7
ui-ngx/src/app/modules/home/components/widget/lib/maps/data-layer/polygons-data-layer.ts

@ -21,7 +21,7 @@ import {
TbMapDatasource, TbPolyData, TbPolygonCoordinates, TbPolygonRawCoordinates
} from '@shared/models/widget/maps/map.models';
import L from 'leaflet';
import { FormattedData } from '@shared/models/widget.models';
import { DataKey, FormattedData } from '@shared/models/widget.models';
import { TbShapesDataLayer } from '@home/components/widget/lib/maps/data-layer/shapes-data-layer';
import { TbMap } from '@home/components/widget/lib/maps/map';
import { Observable } from 'rxjs';
@ -410,9 +410,8 @@ export class TbPolygonsDataLayer extends TbShapesDataLayer<PolygonsDataLayerSett
);
}
protected setupDatasource(datasource: TbMapDatasource): TbMapDatasource {
datasource.dataKeys.push(this.settings.polygonKey);
return datasource;
protected getDataKeys(): DataKey[] {
return [this.settings.polygonKey];
}
protected defaultBaseSettings(map: TbMap<any>): Partial<PolygonsDataLayerSettings> {

32
ui-ngx/src/app/modules/home/components/widget/lib/maps/data-layer/trips-data-layer.ts

@ -351,10 +351,10 @@ class TbTripDataItem extends TbDataLayerItem<TripsDataLayerSettings, TbTripsData
rotationAngle: this.settings.rotateMarker ? this.settings.offsetAngle : 0
};
}
const timeStamp = Object.keys(result);
if (timeline) {
const xKey = this.settings.xKey.label;
const yKey = this.settings.yKey.label;
const timeStamp = Object.keys(result);
for (let i = 0; i < timeStamp.length - 1; i++) {
if (isUndefined(result[timeStamp[i + 1]][xKey]) || isUndefined(result[timeStamp[i + 1]][yKey])) {
for (let j = i + 2; j < timeStamp.length - 1; j++) {
@ -374,6 +374,13 @@ class TbTripDataItem extends TbDataLayerItem<TripsDataLayerSettings, TbTripsData
result[timeStamp[i]].rotationAngle += findRotationAngle(startPoint, endPoint);
}
}
if (this.settings.rotateMarker && timeStamp.length > 1) {
result[timeStamp[timeStamp.length - 1]].rotationAngle = result[timeStamp[timeStamp.length - 2]].rotationAngle;
}
} else if (this.settings.rotateMarker && timeStamp.length > 1) {
const startPoint = this.dataLayer.dataProcessor.extractLocation(result[timeStamp[timeStamp.length - 2]], dsData);
const endPoint = this.dataLayer.dataProcessor.extractLocation(result[timeStamp[timeStamp.length - 1]], dsData);
result[timeStamp[timeStamp.length - 1]].rotationAngle += findRotationAngle(startPoint, endPoint);
}
return result;
}
@ -528,22 +535,33 @@ export class TbTripsDataLayer extends TbMapDataLayer<TripsDataLayerSettings, TbT
};
}
protected setupDatasource(datasource: TbMapDatasource): TbMapDatasource {
datasource.dataKeys = [this.settings.xKey, this.settings.yKey];
protected calculateDataKeys(): DataKey[] {
const dataKeys = [this.settings.xKey, this.settings.yKey];
const additionalKeys = this.allColorSettings().filter(settings => settings.type === DataLayerColorType.range && settings.rangeKey)
.map(settings => settings.rangeKey);
.map(settings => settings.rangeKey);
if (this.settings.additionalDataKeys?.length) {
additionalKeys.push(...this.settings.additionalDataKeys);
}
if (additionalKeys.length) {
const tsKeys = additionalKeys.filter(key => key.type === DataKeyType.timeseries);
dataKeys.push(...tsKeys);
}
return dataKeys;
}
protected calculateLatestDataKeys(): DataKey[] {
const additionalKeys = this.allColorSettings().filter(settings => settings.type === DataLayerColorType.range && settings.rangeKey)
.map(settings => settings.rangeKey);
if (this.settings.additionalDataKeys?.length) {
additionalKeys.push(...this.settings.additionalDataKeys);
}
if (additionalKeys.length) {
const latestKeys = additionalKeys.filter(key => key.type !== DataKeyType.timeseries);
datasource.dataKeys.push(...tsKeys);
if (latestKeys.length) {
datasource.latestDataKeys = latestKeys;
return latestKeys;
}
}
return datasource;
return [];
}
protected allColorSettings(): DataLayerColorSettings[] {

11
ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget.component.html

@ -17,9 +17,14 @@
-->
<div class="tb-map-panel" [style.padding]="padding" [style]="backgroundStyle$ | async">
<div class="tb-map-overlay" [style]="overlayStyle"></div>
<ng-container *ngIf="widgetComponent.dashboardWidget.showWidgetTitlePanel">
<ng-container *ngTemplateOutlet="widgetTitlePanel"></ng-container>
</ng-container>
@if (widgetComponent.dashboardWidget.showWidgetTitlePanel) {
<div class="tb-widget-title-row flex justify-between">
<ng-container *ngTemplateOutlet="widgetComponent.widgetTitlePanel"></ng-container>
<ng-container *ngTemplateOutlet="widgetComponent.widgetHeaderActionsPanel"></ng-container>
</div>
} @else {
<ng-container class="tb-widget-actions-absolute" *ngTemplateOutlet="widgetComponent.widgetHeaderActionsPanel; context: {absolute: true}"></ng-container>
}
<div #mapElement>
</div>
</div>

6
ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget.component.scss

@ -34,4 +34,10 @@
div.tb-widget-title {
padding: 0;
}
.tb-widget-title-row .tb-widget-actions {
margin: 0;
}
.tb-widget-actions.tb-widget-actions-absolute {
z-index: 2;
}
}

4
ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget.component.ts

@ -21,7 +21,6 @@ import {
Input,
OnDestroy,
OnInit,
TemplateRef,
ViewChild,
ViewEncapsulation
} from '@angular/core';
@ -55,9 +54,6 @@ export class MapWidgetComponent implements OnInit, OnDestroy {
@Input()
ctx: WidgetContext;
@Input()
widgetTitlePanel: TemplateRef<any>;
backgroundStyle$: Observable<ComponentStyle>;
overlayStyle: ComponentStyle = {};
padding: string;

7
ui-ngx/src/app/modules/home/components/widget/lib/maps/map.ts

@ -308,7 +308,8 @@ export abstract class TbMap<S extends BaseMapSettings> {
() => {
let datasources: TbMapDatasource[];
for (const layerType of mapDataLayerTypes) {
const typeDatasources = this.latestDataLayers.filter(dl => dl.dataLayerType() === layerType).map(dl => dl.getDatasource());
const typeDatasources = this.latestDataLayers.filter(dl => dl.dataLayerType() === layerType)
.map(dl => dl.getDataSources()).flat();
if (!datasources) {
datasources = typeDatasources;
} else {
@ -349,7 +350,7 @@ export abstract class TbMap<S extends BaseMapSettings> {
);
}
if (this.tripDataLayers.length) {
const tripDatasources = this.tripDataLayers.map(dl => dl.getDatasource());
const tripDatasources = this.tripDataLayers.map(dl => dl.getDataSources()).flat();
const tripDataLayersSubscriptionOptions: WidgetSubscriptionOptions = {
datasources: tripDatasources,
hasDataPageLink: true,
@ -926,7 +927,7 @@ export abstract class TbMap<S extends BaseMapSettings> {
private calculateCurrentTime(minTime: number, maxTime: number): number {
if (minTime !== this.minTime || maxTime !== this.maxTime) {
if (this.minTime >= this.currentTime || isUndefined(this.currentTime)) {
if (this.minTime >= this.currentTime || isUndefined(this.currentTime) || this.currentTime === Infinity) {
return this.minTime;
} else if (this.maxTime <= this.currentTime) {
return this.maxTime;

10
ui-ngx/src/app/modules/home/components/widget/lib/maps/panels/map-timeline-panel.component.ts

@ -202,7 +202,10 @@ export class MapTimelinePanelComponent implements OnInit, OnDestroy {
public moveNext() {
if (this.index < this.maxTimeIndex) {
if (this.settings.snapToRealLocation) {
const anchorIndex = this.findIndex(this.currentTime, this.anchors) + 1;
let anchorIndex = this.findIndex(this.currentTime, this.anchors) + 1;
if (anchorIndex >= this.anchors.length) {
anchorIndex = this.anchors.length - 1;
}
this.index = Math.floor((this.anchors[anchorIndex] - this.minValue) / this.settings.timeStep);
} else {
this.index++;
@ -214,7 +217,10 @@ export class MapTimelinePanelComponent implements OnInit, OnDestroy {
public movePrev() {
if (this.index > this.minTimeIndex) {
if (this.settings.snapToRealLocation) {
const anchorIndex = this.findIndex(this.currentTime, this.anchors) - 1;
let anchorIndex = this.findIndex(this.currentTime, this.anchors) - 1;
if (anchorIndex < 0) {
anchorIndex = 0;
}
this.index = Math.floor((this.anchors[anchorIndex] - this.minValue) / this.settings.timeStep);
} else {
this.index--;

11
ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-widget-settings.component.ts

@ -105,6 +105,17 @@ export class TimeSeriesChartWidgetSettingsComponent extends WidgetSettingsCompon
const params = widgetConfig.typeParameters as any;
if (isDefinedAndNotNull(params.chartType)) {
this.chartType = params.chartType;
} else {
this.chartType = TimeSeriesChartType.default;
}
if (this.timeSeriesChartWidgetSettingsForm) {
const isStateChartType = this.chartType === TimeSeriesChartType.state;
const hasStatesControl = this.timeSeriesChartWidgetSettingsForm.contains('states');
if (isStateChartType && !hasStatesControl) {
this.timeSeriesChartWidgetSettingsForm.addControl('states', this.fb.control(widgetConfig.config.settings.states), { emitEvent: false });
} else if (!isStateChartType && hasStatesControl) {
this.timeSeriesChartWidgetSettingsForm.removeControl('states', { emitEvent: false });
}
}
}

2
ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/widget-action.component.html

@ -37,7 +37,7 @@
<div translate>widget-config.action</div>
<mat-form-field class="flex-1" appearance="outline" subscriptSizing="dynamic">
<mat-select required formControlName="type" placeholder="{{ 'widget-config.set' | translate }}">
<mat-option *ngFor="let actionType of widgetActionTypes" [value]="actionType">
<mat-option *ngFor="let actionType of actionTypes()" [value]="actionType">
{{ widgetActionTypeTranslations.get(widgetActionType[actionType]) | translate }}
</mat-option>
</mat-select>

24
ui-ngx/src/app/modules/home/components/widget/lib/settings/common/action/widget-action.component.ts

@ -26,7 +26,7 @@ import {
ValidatorFn,
Validators
} from '@angular/forms';
import { Component, ElementRef, forwardRef, Input, OnInit, ViewChild } from '@angular/core';
import { Component, computed, ElementRef, forwardRef, input, Input, OnInit, ViewChild } from '@angular/core';
import {
MapItemType,
mapItemTypeTranslationMap,
@ -102,19 +102,16 @@ export class WidgetActionComponent implements ControlValueAccessor, OnInit, Vali
@Input()
actionNames: string[];
@Input()
set additionalWidgetActionTypes(value: WidgetActionType[]) {
if (this.widgetActionFormGroup && !widgetActionTypes.includes(this.widgetActionFormGroup.get('type').value)) {
this.widgetActionFormGroup.get('type').setValue(WidgetActionType.doNothing);
}
if (value?.length) {
this.widgetActionTypes = widgetActionTypes.concat(value);
} else {
this.widgetActionTypes = widgetActionTypes;
additionalWidgetActionTypes = input<WidgetActionType[]>(null);
actionTypes = computed(() => {
const predefinedActionTypes = widgetActionTypes;
if (this.additionalWidgetActionTypes()?.length) {
return predefinedActionTypes.concat(this.additionalWidgetActionTypes());
}
}
return predefinedActionTypes;
});
widgetActionTypes = widgetActionTypes;
widgetActionTypeTranslations = widgetActionTypeTranslationMap;
widgetActionType = WidgetActionType;
@ -191,9 +188,6 @@ export class WidgetActionComponent implements ControlValueAccessor, OnInit, Vali
).subscribe(() => {
this.widgetActionUpdated();
});
if (this.additionalWidgetActionTypes) {
this.widgetActionTypes = this.widgetActionTypes.concat(this.additionalWidgetActionTypes);
}
}
writeValue(widgetAction?: WidgetAction): void {

37
ui-ngx/src/app/modules/home/components/widget/lib/settings/common/filter/filter-select.component.html

@ -15,9 +15,14 @@
limitations under the License.
-->
<mat-form-field [formGroup]="selectFilterFormGroup" class="mat-block" [appearance]="appearance" [subscriptSizing]="subscriptSizing">
<mat-label *ngIf="showLabel">{{ 'filter.filter' | translate }}</mat-label>
<input matInput type="text" placeholder="{{ !showLabel ? ('filter.filter' | translate) : ''}}"
<mat-form-field [formGroup]="selectFilterFormGroup"
[class.tb-inline-field]="inlineField"
[class.flex]="inlineField"
class="mat-block"
[appearance]="inlineField ? 'outline' : appearance"
[subscriptSizing]="inlineField ? 'dynamic' : subscriptSizing">
<mat-label *ngIf="showLabel && !inlineField">{{ 'filter.filter' | translate }}</mat-label>
<input matInput type="text" placeholder="{{ (!showLabel || inlineField) ? ('filter.filter' | translate) : ''}}"
#filterInput
formControlName="filter"
(focusin)="onFocus()"
@ -25,13 +30,23 @@
(keydown)="filterEnter($event)"
(keypress)="filterEnter($event)"
[matAutocomplete]="filterAutocomplete">
<button *ngIf="selectFilterFormGroup.get('filter').value && !disabled"
type="button"
matSuffix mat-icon-button aria-label="Clear"
(click)="clear()">
<mat-icon class="material-icons">close</mat-icon>
</button>
<button *ngIf="!selectFilterFormGroup.get('filter').value && !disabled"
<div matSuffix class="mat-mdc-form-field-icon-suffix flex flex-row">
<button *ngIf="selectFilterFormGroup.get('filter').value && !disabled"
type="button"
mat-icon-button aria-label="Clear"
(click)="clear()">
<mat-icon class="material-icons">close</mat-icon>
</button>
<mat-icon matTooltipPosition="above"
matTooltipClass="tb-error-tooltip"
[matTooltip]="'filter.filter-required' | translate"
*ngIf="inlineField && !modelValue && tbRequired
&& selectFilterFormGroup.get('filter').touched"
class="tb-error">
warning
</mat-icon>
</div>
<button *ngIf="!inlineField && !selectFilterFormGroup.get('filter').value && !disabled"
style="margin-right: 8px;"
type="button"
matSuffix mat-button color="primary"
@ -61,7 +76,7 @@
</div>
</mat-option>
</mat-autocomplete>
<mat-error *ngIf="!modelValue && tbRequired">
<mat-error *ngIf="!inlineField && !modelValue && tbRequired">
{{ 'filter.filter-required' | translate }}
</mat-error>
</mat-form-field>

4
ui-ngx/src/app/modules/home/components/widget/lib/settings/common/filter/filter-select.component.ts

@ -70,6 +70,10 @@ export class FilterSelectComponent implements ControlValueAccessor, OnInit, Afte
@coerceBoolean()
showLabel: boolean;
@Input()
@coerceBoolean()
inlineField: boolean;
@Input()
appearance: MatFormFieldAppearance = 'fill';

80
ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-source-row.component.html

@ -0,0 +1,80 @@
<!--
Copyright © 2016-2025 The Thingsboard Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div [formGroup]="dataSourceFormGroup" class="tb-form-table-row tb-additional-map-data-source-row">
<div class="tb-source-field">
<mat-form-field class="tb-ds-type-field tb-inline-field" appearance="outline" subscriptSizing="dynamic">
<mat-select formControlName="dsType">
<mat-option *ngFor="let type of datasourceTypes" [value]="type">
{{ datasourceTypesTranslations.get(type) | translate }}
</mat-option>
</mat-select>
</mat-form-field>
<mat-form-field
*ngIf="dataSourceFormGroup.get('dsType').value === DatasourceType.function"
class="tb-label-field tb-inline-field" appearance="outline" subscriptSizing="dynamic">
<input matInput formControlName="dsLabel" placeholder="{{ 'widget-config.set' | translate }}">
</mat-form-field>
<tb-entity-autocomplete
class="tb-device-field"
*ngIf="dataSourceFormGroup.get('dsType').value === DatasourceType.device"
required
inlineField
placeholder="{{ 'device.select-device' | translate }}"
[entityType]="EntityType.DEVICE"
formControlName="dsDeviceId">
</tb-entity-autocomplete>
<tb-entity-alias-select
class="tb-entity-alias-field"
*ngIf="dataSourceFormGroup.get('dsType').value === DatasourceType.entity"
inlineField
tbRequired
[aliasController]="context.aliasController"
formControlName="dsEntityAliasId"
[callbacks]="context.callbacks">
</tb-entity-alias-select>
</div>
<tb-data-keys
class="tb-data-keys-field"
formControlName="dataKeys"
inlineField
hideDataKeyColor
hideDataKeyDecimals
hideDataKeyUnits
disableDrag
[required]="true"
[widgetType]="widgetType.latest"
[widget]="context.widget"
[datasourceType]="dataSourceFormGroup.get('dsType').value"
[entityAliasId]="dataSourceFormGroup.get('dsEntityAliasId').value"
[deviceId]="dataSourceFormGroup.get('dsDeviceId').value"
[aliasController]="context.aliasController"
[callbacks]="context.callbacks"
[generateKey]="generateAdditionalDataKey">
</tb-data-keys>
<div class="tb-form-table-row-cell-buttons">
<div class="tb-remove-button">
<button type="button"
mat-icon-button
(click)="dataSourceRemoved.emit()"
matTooltip="{{ 'widgets.maps.data-layer.remove-datasource' | translate }}"
matTooltipPosition="above">
<mat-icon>delete</mat-icon>
</button>
</div>
</div>
</div>

56
ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-source-row.component.scss

@ -0,0 +1,56 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@import '../../../../../../../../../scss/constants';
.tb-form-table-row.tb-additional-map-data-source-row {
.tb-source-field {
flex: 1 1 50%;
display: flex;
gap: 12px;
.tb-ds-type-field, .tb-label-field, .tb-device-field, .tb-entity-alias-field {
flex: 1;
}
}
.tb-data-keys-field {
flex: 1 1 50%;
min-width: 0;
}
.tb-remove-button {
width: 40px;
min-width: 40px;
}
@media #{$mat-lt-lg} {
.tb-source-field {
flex-direction: column;
flex: 1 1 30%;
}
.tb-data-keys-field {
flex: 1 1 70%;
}
@media #{$mat-lt-md} {
.tb-source-field {
flex: 1 1 50%;
}
.tb-data-keys-field {
flex: 1 1 50%;
}
}
}
}

207
ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-source-row.component.ts

@ -0,0 +1,207 @@
///
/// Copyright © 2016-2025 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import {
ChangeDetectorRef,
Component,
DestroyRef,
EventEmitter,
forwardRef,
Input,
OnInit,
Output,
ViewEncapsulation
} from '@angular/core';
import {
ControlValueAccessor,
NG_VALUE_ACCESSOR,
UntypedFormBuilder,
UntypedFormGroup,
Validators
} from '@angular/forms';
import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
import { AdditionalMapDataSourceSettings } from '@shared/models/widget/maps/map.models';
import { DataKey, DatasourceType, datasourceTypeTranslationMap, widgetType } from '@shared/models/widget.models';
import { EntityType } from '@shared/models/entity-type.models';
import { DataKeyType } from '@shared/models/telemetry/telemetry.models';
import { genNextLabelForDataKeys } from '@core/utils';
import { MapSettingsContext } from '@home/components/widget/lib/settings/common/map/map-settings.component.models';
@Component({
selector: 'tb-additional-map-data-source-row',
templateUrl: './additional-map-data-source-row.component.html',
styleUrls: ['./additional-map-data-source-row.component.scss'],
providers: [
{
provide: NG_VALUE_ACCESSOR,
useExisting: forwardRef(() => AdditionalMapDataSourceRowComponent),
multi: true
}
],
encapsulation: ViewEncapsulation.None
})
export class AdditionalMapDataSourceRowComponent implements ControlValueAccessor, OnInit {
DatasourceType = DatasourceType;
DataKeyType = DataKeyType;
EntityType = EntityType;
widgetType = widgetType;
datasourceTypes: Array<DatasourceType> = [];
datasourceTypesTranslations = datasourceTypeTranslationMap;
@Input()
disabled: boolean;
@Input()
context: MapSettingsContext;
@Output()
dataSourceRemoved = new EventEmitter();
dataSourceFormGroup: UntypedFormGroup;
generateAdditionalDataKey = this.generateDataKey.bind(this);
modelValue: AdditionalMapDataSourceSettings;
private propagateChange = (_val: any) => {};
constructor(private fb: UntypedFormBuilder,
private cd: ChangeDetectorRef,
private destroyRef: DestroyRef) {
}
ngOnInit() {
if (this.context.functionsOnly) {
this.datasourceTypes = [DatasourceType.function];
} else {
this.datasourceTypes = [DatasourceType.function, DatasourceType.device, DatasourceType.entity];
}
this.dataSourceFormGroup = this.fb.group({
dsType: [null, [Validators.required]],
dsLabel: [null, []],
dsDeviceId: [null, [Validators.required]],
dsEntityAliasId: [null, [Validators.required]],
dataKeys: [null, [Validators.required]]
});
this.dataSourceFormGroup.valueChanges.pipe(
takeUntilDestroyed(this.destroyRef)
).subscribe(
() => this.updateModel()
);
this.dataSourceFormGroup.get('dsType').valueChanges.pipe(
takeUntilDestroyed(this.destroyRef)
).subscribe(
(newDsType: DatasourceType) => this.onDsTypeChanged(newDsType)
);
}
registerOnChange(fn: any): void {
this.propagateChange = fn;
}
registerOnTouched(_fn: any): void {
}
setDisabledState(isDisabled: boolean): void {
this.disabled = isDisabled;
if (isDisabled) {
this.dataSourceFormGroup.disable({emitEvent: false});
} else {
this.dataSourceFormGroup.enable({emitEvent: false});
this.updateValidators();
}
}
writeValue(value: AdditionalMapDataSourceSettings): void {
this.modelValue = value;
this.dataSourceFormGroup.patchValue(
{
dsType: value?.dsType,
dsLabel: value?.dsLabel,
dsDeviceId: value?.dsDeviceId,
dsEntityAliasId: value?.dsEntityAliasId,
dataKeys: value?.dataKeys
}, {emitEvent: false}
);
this.updateValidators();
this.cd.markForCheck();
}
private generateDataKey(key: DataKey): DataKey {
const dataKey = this.context.callbacks.generateDataKey(key.name, key.type, null, false, null);
const dataKeys: DataKey[] = this.dataSourceFormGroup.get('dataKeys').value || [];
dataKey.label = genNextLabelForDataKeys(dataKey.label, dataKeys);
return dataKey;
}
private onDsTypeChanged(newDsType: DatasourceType) {
let updateModel = false;
const dataKeys: DataKey[] = this.dataSourceFormGroup.get('dataKeys').value;
if (dataKeys?.length) {
for (const key of dataKeys) {
updateModel = this.updateDataKeyToNewDsType(key, newDsType) || updateModel;
}
if (updateModel) {
this.dataSourceFormGroup.get('dataKeys').patchValue(dataKeys, {emitEvent: false});
}
}
this.updateValidators();
if (updateModel) {
this.updateModel();
}
}
private updateDataKeyToNewDsType(dataKey: DataKey, newDsType: DatasourceType): boolean {
if (newDsType === DatasourceType.function) {
if (dataKey.type !== DataKeyType.function) {
dataKey.type = DataKeyType.function;
return true;
}
} else {
if (dataKey.type === DataKeyType.function) {
dataKey.type = DataKeyType.attribute;
return true;
}
}
return false;
}
private updateValidators() {
const dsType: DatasourceType = this.dataSourceFormGroup.get('dsType').value;
if (dsType === DatasourceType.function) {
this.dataSourceFormGroup.get('dsLabel').enable({emitEvent: false});
this.dataSourceFormGroup.get('dsDeviceId').disable({emitEvent: false});
this.dataSourceFormGroup.get('dsEntityAliasId').disable({emitEvent: false});
} else if (dsType === DatasourceType.device) {
this.dataSourceFormGroup.get('dsLabel').disable({emitEvent: false});
this.dataSourceFormGroup.get('dsDeviceId').enable({emitEvent: false});
this.dataSourceFormGroup.get('dsEntityAliasId').disable({emitEvent: false});
} else {
this.dataSourceFormGroup.get('dsLabel').disable({emitEvent: false});
this.dataSourceFormGroup.get('dsDeviceId').disable({emitEvent: false});
this.dataSourceFormGroup.get('dsEntityAliasId').enable({emitEvent: false});
}
}
private updateModel() {
this.modelValue = {...this.modelValue, ...this.dataSourceFormGroup.value};
this.propagateChange(this.modelValue);
}
}

43
ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-sources.component.html

@ -0,0 +1,43 @@
<!--
Copyright © 2016-2025 The Thingsboard Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div class="tb-form-panel no-padding no-border">
<div class="tb-form-table tb-additional-map-data-sources">
<div class="tb-form-table-header no-padding-right">
<div class="tb-form-table-header-cell tb-source-header" translate>widgets.maps.data-layer.source</div>
<div class="tb-form-table-header-cell tb-data-keys-header" translate>widgets.maps.data-layer.data-keys</div>
<div class="tb-form-table-header-cell tb-actions-header"></div>
</div>
<div *ngIf="dataSourcesFormArray().controls.length; else noDataSources" class="tb-form-table-body">
<div *ngFor="let dataSourceControl of dataSourcesFormArray().controls; trackBy: trackByDataSource; let $index = index;">
<tb-additional-map-data-source-row class="flex-1"
[context]="context"
[formControl]="dataSourceControl"
(dataSourceRemoved)="removeDataSource($index)">
</tb-additional-map-data-source-row>
</div>
</div>
</div>
<div>
<button type="button" mat-stroked-button color="primary" (click)="addDataSource()">
{{ 'widgets.maps.data-layer.add-datasource' | translate }}
</button>
</div>
</div>
<ng-template #noDataSources>
<span class="tb-prompt flex items-center justify-center">{{ 'widgets.maps.data-layer.no-datasources' | translate }}</span>
</ng-template>

53
ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-sources.component.scss

@ -0,0 +1,53 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@import '../../../../../../../../../scss/constants';
.tb-additional-map-data-sources {
.tb-form-table-header-cell {
&.tb-source-header {
flex: 1 1 50%;
}
&.tb-data-keys-header {
flex: 1 1 50%;
}
&.tb-actions-header {
width: 40px;
min-width: 40px;
}
@media #{$mat-lt-lg} {
&.tb-source-header {
flex: 1 1 30%;
}
&.tb-data-keys-header {
flex: 1 1 70%;
}
@media #{$mat-lt-md} {
&.tb-source-header {
flex: 1 1 50%;
}
&.tb-data-keys-header {
flex: 1 1 50%;
}
}
}
}
.tb-form-table-body {
tb-additional-map-data-source-row {
overflow: hidden;
}
}
}

147
ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/additional-map-data-sources.component.ts

@ -0,0 +1,147 @@
///
/// Copyright © 2016-2025 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import { Component, DestroyRef, forwardRef, Input, OnInit, ViewEncapsulation } from '@angular/core';
import {
AbstractControl,
ControlValueAccessor,
NG_VALIDATORS,
NG_VALUE_ACCESSOR,
UntypedFormArray,
UntypedFormBuilder,
UntypedFormControl,
UntypedFormGroup,
Validator
} from '@angular/forms';
import { mergeDeep } from '@core/utils';
import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
import {
AdditionalMapDataSourceSettings,
additionalMapDataSourceValid,
additionalMapDataSourceValidator,
defaultAdditionalMapDataSourceSettings
} from '@shared/models/widget/maps/map.models';
import { MapSettingsContext } from '@home/components/widget/lib/settings/common/map/map-settings.component.models';
@Component({
selector: 'tb-additional-map-data-sources',
templateUrl: './additional-map-data-sources.component.html',
styleUrls: ['./additional-map-data-sources.component.scss'],
providers: [
{
provide: NG_VALUE_ACCESSOR,
useExisting: forwardRef(() => AdditionalMapDataSourcesComponent),
multi: true
},
{
provide: NG_VALIDATORS,
useExisting: forwardRef(() => AdditionalMapDataSourcesComponent),
multi: true
}
],
encapsulation: ViewEncapsulation.None
})
export class AdditionalMapDataSourcesComponent implements ControlValueAccessor, OnInit, Validator {
@Input()
disabled: boolean;
@Input()
context: MapSettingsContext;
dataSourcesFormGroup: UntypedFormGroup;
private propagateChange = (_val: any) => {};
constructor(private fb: UntypedFormBuilder,
private destroyRef: DestroyRef) {
}
ngOnInit() {
this.dataSourcesFormGroup = this.fb.group({
dataSources: [this.fb.array([]), []]
});
this.dataSourcesFormGroup.valueChanges.pipe(
takeUntilDestroyed(this.destroyRef)
).subscribe(
() => {
let dataSources: AdditionalMapDataSourceSettings[] = this.dataSourcesFormGroup.get('dataSources').value;
if (dataSources) {
dataSources = dataSources.filter(dataSource => additionalMapDataSourceValid(dataSource));
}
this.propagateChange(dataSources);
}
);
}
registerOnChange(fn: any): void {
this.propagateChange = fn;
}
registerOnTouched(fn: any): void {
}
setDisabledState(isDisabled: boolean): void {
this.disabled = isDisabled;
if (isDisabled) {
this.dataSourcesFormGroup.disable({emitEvent: false});
} else {
this.dataSourcesFormGroup.enable({emitEvent: false});
}
}
writeValue(value: AdditionalMapDataSourceSettings[] | undefined): void {
const dataSources: AdditionalMapDataSourceSettings[] = value || [];
this.dataSourcesFormGroup.setControl('dataSources', this.prepareDataSourcesFormArray(dataSources), {emitEvent: false});
}
public validate(c: UntypedFormControl) {
const valid = this.dataSourcesFormGroup.valid;
return valid ? null : {
dataSources: {
valid: false,
},
};
}
dataSourcesFormArray(): UntypedFormArray {
return this.dataSourcesFormGroup.get('dataSources') as UntypedFormArray;
}
trackByDataSource(index: number, dataSourceControl: AbstractControl): any {
return dataSourceControl;
}
removeDataSource(index: number) {
(this.dataSourcesFormGroup.get('dataSources') as UntypedFormArray).removeAt(index);
}
addDataSource() {
const dataSource = mergeDeep<AdditionalMapDataSourceSettings>({} as AdditionalMapDataSourceSettings,
defaultAdditionalMapDataSourceSettings(this.context.functionsOnly));
const dataSourcesArray = this.dataSourcesFormGroup.get('dataSources') as UntypedFormArray;
const dataSourceControl = this.fb.control(dataSource, [additionalMapDataSourceValidator]);
dataSourcesArray.push(dataSourceControl);
}
private prepareDataSourcesFormArray(dataSources: AdditionalMapDataSourceSettings[]): UntypedFormArray {
const dataSourcesControls: Array<AbstractControl> = [];
dataSources.forEach((dataSource) => {
dataSourcesControls.push(this.fb.control(dataSource, [additionalMapDataSourceValidator]));
});
return this.fb.array(dataSourcesControls);
}
}

2
ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/data-layer-color-settings-panel.component.html

@ -39,7 +39,7 @@
<div class="tb-data-layer-color-settings-panel-body" [class.!hidden]="colorSettingsFormGroup.get('type').value !== DataLayerColorType.constant">
</div>
<div class="tb-data-layer-color-settings-panel-body" [class.!hidden]="colorSettingsFormGroup.get('type').value !== DataLayerColorType.range">
<tb-data-key-input
<tb-data-key-input style="padding-top: 6px;"
inlineField="false"
appearance="outline"
subscriptSizing="fixed"

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save