diff --git a/application/src/main/data/json/system/scada_symbols/apartments-hp.svg b/application/src/main/data/json/system/scada_symbols/apartments-hp.svg index 9e2c965f79..97ac492626 100644 --- a/application/src/main/data/json/system/scada_symbols/apartments-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/apartments-hp.svg @@ -330,7 +330,7 @@ } ] }]]> - + diff --git a/application/src/main/data/json/system/scada_symbols/bottom-light-bulb-hp.svg b/application/src/main/data/json/system/scada_symbols/bottom-light-bulb-hp.svg index 4f0603b904..db107240a0 100644 --- a/application/src/main/data/json/system/scada_symbols/bottom-light-bulb-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/bottom-light-bulb-hp.svg @@ -332,7 +332,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/conical-tank.svg b/application/src/main/data/json/system/scada_symbols/conical-tank.svg index 7ae8f41934..6b37adc52e 100644 --- a/application/src/main/data/json/system/scada_symbols/conical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/conical-tank.svg @@ -371,7 +371,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/consumers-hp.svg b/application/src/main/data/json/system/scada_symbols/consumers-hp.svg index e4005d7ae2..8333d6bf55 100644 --- a/application/src/main/data/json/system/scada_symbols/consumers-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/consumers-hp.svg @@ -332,7 +332,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/dynamic-horizontal-scale-hp.svg b/application/src/main/data/json/system/scada_symbols/dynamic-horizontal-scale-hp.svg index e782b07609..b3506ae54f 100644 --- a/application/src/main/data/json/system/scada_symbols/dynamic-horizontal-scale-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/dynamic-horizontal-scale-hp.svg @@ -791,5 +791,5 @@ - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg b/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg index 960abed340..3c52dae1c3 100644 --- a/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg @@ -791,5 +791,5 @@ 26 - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/electrical-distribution-board-hp.svg b/application/src/main/data/json/system/scada_symbols/electrical-distribution-board-hp.svg index 0fcfe4dde0..6e4175c6d7 100644 --- a/application/src/main/data/json/system/scada_symbols/electrical-distribution-board-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/electrical-distribution-board-hp.svg @@ -318,7 +318,7 @@ }]]> - + diff --git a/application/src/main/data/json/system/scada_symbols/filter-hp.svg b/application/src/main/data/json/system/scada_symbols/filter-hp.svg index 58ff3951e5..1f4516723e 100644 --- a/application/src/main/data/json/system/scada_symbols/filter-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/filter-hp.svg @@ -273,7 +273,7 @@ } ] }]]> - + diff --git a/application/src/main/data/json/system/scada_symbols/four-rate-energy-meter-hp.svg b/application/src/main/data/json/system/scada_symbols/four-rate-energy-meter-hp.svg index de4fb13836..bba67e5fe3 100644 --- a/application/src/main/data/json/system/scada_symbols/four-rate-energy-meter-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/four-rate-energy-meter-hp.svg @@ -862,7 +862,7 @@ } ] }]]> -T1T2T3Export000223000223000223000223kWh +T1T2T3Export000223000223000223000223kWh diff --git a/application/src/main/data/json/system/scada_symbols/gas-preventer-hp.svg b/application/src/main/data/json/system/scada_symbols/gas-preventer-hp.svg index ea5a479eda..487b5e54d9 100644 --- a/application/src/main/data/json/system/scada_symbols/gas-preventer-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/gas-preventer-hp.svg @@ -347,7 +347,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/heat-exchanger-hp.svg b/application/src/main/data/json/system/scada_symbols/heat-exchanger-hp.svg index 797220ea96..c1c3d4647e 100644 --- a/application/src/main/data/json/system/scada_symbols/heat-exchanger-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/heat-exchanger-hp.svg @@ -349,7 +349,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/heat-pump-hp.svg b/application/src/main/data/json/system/scada_symbols/heat-pump-hp.svg index 1d7babc3b0..eed54682cd 100644 --- a/application/src/main/data/json/system/scada_symbols/heat-pump-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/heat-pump-hp.svg @@ -587,7 +587,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/horizontal-tank-hp.svg b/application/src/main/data/json/system/scada_symbols/horizontal-tank-hp.svg index 18f194684b..bb6f5e55a9 100644 --- a/application/src/main/data/json/system/scada_symbols/horizontal-tank-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/horizontal-tank-hp.svg @@ -502,7 +502,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/house-hp.svg b/application/src/main/data/json/system/scada_symbols/house-hp.svg index d95c47bb8b..e3556d4d9a 100644 --- a/application/src/main/data/json/system/scada_symbols/house-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/house-hp.svg @@ -335,7 +335,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/industrial-fuel-generator-hp.svg b/application/src/main/data/json/system/scada_symbols/industrial-fuel-generator-hp.svg index 2b7a476a92..e0ee85b8cf 100644 --- a/application/src/main/data/json/system/scada_symbols/industrial-fuel-generator-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/industrial-fuel-generator-hp.svg @@ -344,7 +344,7 @@ } ] }]]> - + diff --git a/application/src/main/data/json/system/scada_symbols/large-horizontal-separator-hp.svg b/application/src/main/data/json/system/scada_symbols/large-horizontal-separator-hp.svg index 79f6745a73..8179549c54 100644 --- a/application/src/main/data/json/system/scada_symbols/large-horizontal-separator-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/large-horizontal-separator-hp.svg @@ -336,7 +336,7 @@ } ] }]]> - + diff --git a/application/src/main/data/json/system/scada_symbols/large-inverter-hp.svg b/application/src/main/data/json/system/scada_symbols/large-inverter-hp.svg index 11bc5da0a6..bf18d64040 100644 --- a/application/src/main/data/json/system/scada_symbols/large-inverter-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/large-inverter-hp.svg @@ -564,7 +564,7 @@ } ] }]]> - + diff --git a/application/src/main/data/json/system/scada_symbols/large-stand-cylindrical-tank.svg b/application/src/main/data/json/system/scada_symbols/large-stand-cylindrical-tank.svg index 11bd47916f..09c0e2a9e1 100644 --- a/application/src/main/data/json/system/scada_symbols/large-stand-cylindrical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/large-stand-cylindrical-tank.svg @@ -565,7 +565,7 @@ 1660 gal - + diff --git a/application/src/main/data/json/system/scada_symbols/large-stand-vertical-tank.svg b/application/src/main/data/json/system/scada_symbols/large-stand-vertical-tank.svg index d9c05bde40..be8b1207a0 100644 --- a/application/src/main/data/json/system/scada_symbols/large-stand-vertical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/large-stand-vertical-tank.svg @@ -569,7 +569,7 @@ 1660 gal - + diff --git a/application/src/main/data/json/system/scada_symbols/large-vertical-separator-hp.svg b/application/src/main/data/json/system/scada_symbols/large-vertical-separator-hp.svg index 99ff420530..1951b741bf 100644 --- a/application/src/main/data/json/system/scada_symbols/large-vertical-separator-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/large-vertical-separator-hp.svg @@ -336,7 +336,7 @@ } ] }]]> - + diff --git a/application/src/main/data/json/system/scada_symbols/leak-sensor.svg b/application/src/main/data/json/system/scada_symbols/leak-sensor.svg index 0fce8be608..813116a1f5 100644 --- a/application/src/main/data/json/system/scada_symbols/leak-sensor.svg +++ b/application/src/main/data/json/system/scada_symbols/leak-sensor.svg @@ -135,7 +135,7 @@ } ] }]]> - + diff --git a/application/src/main/data/json/system/scada_symbols/low-voltage-tower-hp.svg b/application/src/main/data/json/system/scada_symbols/low-voltage-tower-hp.svg index 812e616433..002bd9cfae 100644 --- a/application/src/main/data/json/system/scada_symbols/low-voltage-tower-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/low-voltage-tower-hp.svg @@ -264,7 +264,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/meter.svg b/application/src/main/data/json/system/scada_symbols/meter.svg index 02b2833133..d5fa9f7cd7 100644 --- a/application/src/main/data/json/system/scada_symbols/meter.svg +++ b/application/src/main/data/json/system/scada_symbols/meter.svg @@ -713,7 +713,7 @@ 37% - + diff --git a/application/src/main/data/json/system/scada_symbols/oil-pump-hp.svg b/application/src/main/data/json/system/scada_symbols/oil-pump-hp.svg index e7dfa8f0c8..b1ff6ea2c0 100644 --- a/application/src/main/data/json/system/scada_symbols/oil-pump-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/oil-pump-hp.svg @@ -343,7 +343,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/pool-hp.svg b/application/src/main/data/json/system/scada_symbols/pool-hp.svg index 925ea8906a..0ce78af7d9 100644 --- a/application/src/main/data/json/system/scada_symbols/pool-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/pool-hp.svg @@ -501,7 +501,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/pool.svg b/application/src/main/data/json/system/scada_symbols/pool.svg index f5d0bd7ad7..6f8b12737c 100644 --- a/application/src/main/data/json/system/scada_symbols/pool.svg +++ b/application/src/main/data/json/system/scada_symbols/pool.svg @@ -292,7 +292,7 @@ 1660 gal - + diff --git a/application/src/main/data/json/system/scada_symbols/power-socket-hp.svg b/application/src/main/data/json/system/scada_symbols/power-socket-hp.svg index 5524b3bb14..2331242475 100644 --- a/application/src/main/data/json/system/scada_symbols/power-socket-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/power-socket-hp.svg @@ -357,7 +357,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/short-vertical-tank-hp.svg b/application/src/main/data/json/system/scada_symbols/short-vertical-tank-hp.svg index 046c5c5802..cd15a16577 100644 --- a/application/src/main/data/json/system/scada_symbols/short-vertical-tank-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/short-vertical-tank-hp.svg @@ -502,7 +502,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/simple-horizontal-scale-hp.svg b/application/src/main/data/json/system/scada_symbols/simple-horizontal-scale-hp.svg index 26cee5cd30..1b91a0cd4a 100644 --- a/application/src/main/data/json/system/scada_symbols/simple-horizontal-scale-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/simple-horizontal-scale-hp.svg @@ -719,5 +719,5 @@ - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg b/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg index fa0c0fc4d1..7b8e7299b8 100644 --- a/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg @@ -719,5 +719,5 @@ 26 - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/small-cylindrical-tank.svg b/application/src/main/data/json/system/scada_symbols/small-cylindrical-tank.svg index d4ddda1895..1d6785d0c1 100644 --- a/application/src/main/data/json/system/scada_symbols/small-cylindrical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/small-cylindrical-tank.svg @@ -537,7 +537,7 @@ 1660 gal - + diff --git a/application/src/main/data/json/system/scada_symbols/small-horizontal-separator-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/small-horizontal-separator-connector-hp.svg index dedc4dc81e..abb6d9b4ab 100644 --- a/application/src/main/data/json/system/scada_symbols/small-horizontal-separator-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/small-horizontal-separator-connector-hp.svg @@ -328,7 +328,7 @@ } ] }]]> - + diff --git a/application/src/main/data/json/system/scada_symbols/small-left-meter.svg b/application/src/main/data/json/system/scada_symbols/small-left-meter.svg index a480412366..129006ffd7 100644 --- a/application/src/main/data/json/system/scada_symbols/small-left-meter.svg +++ b/application/src/main/data/json/system/scada_symbols/small-left-meter.svg @@ -713,5 +713,5 @@ 37% - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/small-vertical-separator-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/small-vertical-separator-connector-hp.svg index b248996612..e9cc074cb6 100644 --- a/application/src/main/data/json/system/scada_symbols/small-vertical-separator-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/small-vertical-separator-connector-hp.svg @@ -328,7 +328,7 @@ } ] }]]> - + diff --git a/application/src/main/data/json/system/scada_symbols/small-vertical-separator-hp.svg b/application/src/main/data/json/system/scada_symbols/small-vertical-separator-hp.svg index 25aec080b3..0c7455e534 100644 --- a/application/src/main/data/json/system/scada_symbols/small-vertical-separator-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/small-vertical-separator-hp.svg @@ -328,7 +328,7 @@ } ] }]]> - + diff --git a/application/src/main/data/json/system/scada_symbols/spherical-tank.svg b/application/src/main/data/json/system/scada_symbols/spherical-tank.svg index cba49c63e0..f5d679fa96 100644 --- a/application/src/main/data/json/system/scada_symbols/spherical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/spherical-tank.svg @@ -572,7 +572,7 @@ 1660 gal - + diff --git a/application/src/main/data/json/system/scada_symbols/stand-cylindrical-tank.svg b/application/src/main/data/json/system/scada_symbols/stand-cylindrical-tank.svg index a78060dfaf..f5b8e8a892 100644 --- a/application/src/main/data/json/system/scada_symbols/stand-cylindrical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/stand-cylindrical-tank.svg @@ -567,7 +567,7 @@ 1660 gal - + diff --git a/application/src/main/data/json/system/scada_symbols/stand-vertical-short-tank.svg b/application/src/main/data/json/system/scada_symbols/stand-vertical-short-tank.svg index 96df175589..0d56901bfe 100644 --- a/application/src/main/data/json/system/scada_symbols/stand-vertical-short-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/stand-vertical-short-tank.svg @@ -1364,5 +1364,5 @@ - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/top-light-bulb-hp.svg b/application/src/main/data/json/system/scada_symbols/top-light-bulb-hp.svg index ed6855884a..fb419bac2a 100644 --- a/application/src/main/data/json/system/scada_symbols/top-light-bulb-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/top-light-bulb-hp.svg @@ -332,7 +332,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/turbine-hp.svg b/application/src/main/data/json/system/scada_symbols/turbine-hp.svg index d798b09477..2e0e176cfd 100644 --- a/application/src/main/data/json/system/scada_symbols/turbine-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/turbine-hp.svg @@ -363,7 +363,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/vertical-energy-system-controller-hp.svg b/application/src/main/data/json/system/scada_symbols/vertical-energy-system-controller-hp.svg index cebe949c36..6da68556a2 100644 --- a/application/src/main/data/json/system/scada_symbols/vertical-energy-system-controller-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/vertical-energy-system-controller-hp.svg @@ -364,7 +364,7 @@ } ] }]]> -Connected +Connected diff --git a/application/src/main/data/json/system/scada_symbols/vertical-tank-hp.svg b/application/src/main/data/json/system/scada_symbols/vertical-tank-hp.svg index 491fb4477e..9dee6cc2af 100644 --- a/application/src/main/data/json/system/scada_symbols/vertical-tank-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/vertical-tank-hp.svg @@ -502,7 +502,7 @@ - + diff --git a/application/src/main/data/json/system/scada_symbols/voltage-stabilizer-hp.svg b/application/src/main/data/json/system/scada_symbols/voltage-stabilizer-hp.svg index aafc2af5a0..2ccad581d4 100644 --- a/application/src/main/data/json/system/scada_symbols/voltage-stabilizer-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/voltage-stabilizer-hp.svg @@ -570,7 +570,7 @@ } ] }]]> -220230inout +220230inout diff --git a/application/src/main/data/json/system/scada_symbols/wind-turbine-cluster-hp.svg b/application/src/main/data/json/system/scada_symbols/wind-turbine-cluster-hp.svg index 74855dd35a..bdc4d23803 100644 --- a/application/src/main/data/json/system/scada_symbols/wind-turbine-cluster-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/wind-turbine-cluster-hp.svg @@ -349,14 +349,14 @@ - + - + diff --git a/application/src/main/data/json/system/scada_symbols/wind-turbine-hp.svg b/application/src/main/data/json/system/scada_symbols/wind-turbine-hp.svg index a4282c16e7..b2c65988da 100644 --- a/application/src/main/data/json/system/scada_symbols/wind-turbine-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/wind-turbine-hp.svg @@ -346,7 +346,7 @@ - + diff --git a/application/src/main/data/json/system/widget_types/bar_chart.json b/application/src/main/data/json/system/widget_types/bar_chart.json index 01f16d96b5..58f6244fd3 100644 --- a/application/src/main/data/json/system/widget_types/bar_chart.json +++ b/application/src/main/data/json/system/widget_types/bar_chart.json @@ -9,9 +9,9 @@ "sizeX": 8, "sizeY": 5, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'bar',\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('bar'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'bar',\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('bar'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n", "settingsSchema": "{}", "dataKeySettingsSchema": "{}", "latestDataKeySettingsSchema": "{}", diff --git a/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json b/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json index a2fc5f5c9d..6972e859b0 100644 --- a/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json +++ b/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json @@ -9,9 +9,9 @@ "sizeX": 8, "sizeY": 5, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": ".legend {\n font-size: 13px;\n line-height: 10px;\n}\n\n.legend table { \n border-spacing: 0px;\n border-collapse: separate;\n}\n\n.mouse-events .flot-overlay {\n cursor: crosshair; \n}\n\n", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'humidity', label: 'Humidity', type: 'timeseries' }];\n }\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'humidity', label: 'Humidity', type: 'timeseries' }];\n }\n };\n}\n", "settingsSchema": "{}", "dataKeySettingsSchema": "{}", "latestDataKeySettingsSchema": "{}", diff --git a/application/src/main/data/json/system/widget_types/bars.json b/application/src/main/data/json/system/widget_types/bars.json index 481288da9b..ce640c95b1 100644 --- a/application/src/main/data/json/system/widget_types/bars.json +++ b/application/src/main/data/json/system/widget_types/bars.json @@ -9,9 +9,9 @@ "sizeX": 5, "sizeY": 4, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n", "settingsSchema": "", "dataKeySettingsSchema": "", "settingsDirective": "tb-bar-chart-widget-settings", diff --git a/application/src/main/data/json/system/widget_types/doughnut.json b/application/src/main/data/json/system/widget_types/doughnut.json index 5065100a91..bf03a75b09 100644 --- a/application/src/main/data/json/system/widget_types/doughnut.json +++ b/application/src/main/data/json/system/widget_types/doughnut.json @@ -9,9 +9,9 @@ "sizeX": 4, "sizeY": 3, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.doughnutWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.doughnutWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '380px',\n previewHeight: '300px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind power', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar power', type: 'timeseries', color: '#FF4D5A' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.doughnutWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.doughnutWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '380px',\n previewHeight: '300px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind power', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar power', type: 'timeseries', color: '#FF4D5A' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n", "settingsSchema": "", "dataKeySettingsSchema": "", "settingsDirective": "tb-doughnut-widget-settings", diff --git a/application/src/main/data/json/system/widget_types/horizontal_doughnut.json b/application/src/main/data/json/system/widget_types/horizontal_doughnut.json index a6e7f4d3bd..050984268c 100644 --- a/application/src/main/data/json/system/widget_types/horizontal_doughnut.json +++ b/application/src/main/data/json/system/widget_types/horizontal_doughnut.json @@ -9,9 +9,9 @@ "sizeX": 4, "sizeY": 2.5, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.doughnutWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.doughnutWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '380px',\n previewHeight: '220px',\n embedTitlePanel: true,\n horizontal: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind power', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar power', type: 'timeseries', color: '#FF4D5A' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.doughnutWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.doughnutWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '380px',\n previewHeight: '220px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n horizontal: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind power', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar power', type: 'timeseries', color: '#FF4D5A' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n", "settingsSchema": "", "dataKeySettingsSchema": "", "settingsDirective": "tb-doughnut-widget-settings", diff --git a/application/src/main/data/json/system/widget_types/image_map.json b/application/src/main/data/json/system/widget_types/image_map.json index 2a4fe71696..991c0b463f 100644 --- a/application/src/main/data/json/system/widget_types/image_map.json +++ b/application/src/main/data/json/system/widget_types/image_map.json @@ -9,9 +9,9 @@ "sizeX": 8.5, "sizeY": 6, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}\n", "settingsForm": [], "dataKeySettingsForm": [], "settingsDirective": "tb-map-widget-settings", diff --git a/application/src/main/data/json/system/widget_types/line_chart.json b/application/src/main/data/json/system/widget_types/line_chart.json index 3ef1b83c03..0468c42f8e 100644 --- a/application/src/main/data/json/system/widget_types/line_chart.json +++ b/application/src/main/data/json/system/widget_types/line_chart.json @@ -9,9 +9,9 @@ "sizeX": 8, "sizeY": 5, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'line',\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('line'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'line',\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('line'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n", "settingsSchema": "{}", "dataKeySettingsSchema": "{}", "latestDataKeySettingsSchema": "{}", diff --git a/application/src/main/data/json/system/widget_types/map.json b/application/src/main/data/json/system/widget_types/map.json index b9069d8af6..e498922d36 100644 --- a/application/src/main/data/json/system/widget_types/map.json +++ b/application/src/main/data/json/system/widget_types/map.json @@ -9,9 +9,9 @@ "sizeX": 8.5, "sizeY": 6, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}\n", "settingsForm": [], "dataKeySettingsForm": [], "settingsDirective": "tb-map-widget-settings", diff --git a/application/src/main/data/json/system/widget_types/pie.json b/application/src/main/data/json/system/widget_types/pie.json index 01c96a5077..94e02d06ef 100644 --- a/application/src/main/data/json/system/widget_types/pie.json +++ b/application/src/main/data/json/system/widget_types/pie.json @@ -9,9 +9,9 @@ "sizeX": 5, "sizeY": 4, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.pieChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.pieChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\nself.actionSources = function() {\n return {\n 'sliceClick': {\n name: 'widget-action.pie-slice-click',\n multiple: false\n }\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.pieChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.pieChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\nself.actionSources = function() {\n return {\n 'sliceClick': {\n name: 'widget-action.pie-slice-click',\n multiple: false\n }\n };\n}\n", "settingsSchema": "", "dataKeySettingsSchema": "", "settingsDirective": "tb-pie-chart-widget-settings", diff --git a/application/src/main/data/json/system/widget_types/point_chart.json b/application/src/main/data/json/system/widget_types/point_chart.json index a3f1ed5f60..74cd137665 100644 --- a/application/src/main/data/json/system/widget_types/point_chart.json +++ b/application/src/main/data/json/system/widget_types/point_chart.json @@ -9,9 +9,9 @@ "sizeX": 8, "sizeY": 5, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'point',\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('point'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n chartType: 'point',\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('point'),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n", "settingsSchema": "{}", "dataKeySettingsSchema": "{}", "latestDataKeySettingsSchema": "{}", diff --git a/application/src/main/data/json/system/widget_types/polar_area.json b/application/src/main/data/json/system/widget_types/polar_area.json index 2bdce5d9ad..36fcaee465 100644 --- a/application/src/main/data/json/system/widget_types/polar_area.json +++ b/application/src/main/data/json/system/widget_types/polar_area.json @@ -9,9 +9,9 @@ "sizeX": 5, "sizeY": 4, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.polarAreaChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.polarAreaChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.polarAreaChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.polarAreaChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n", "settingsSchema": "", "dataKeySettingsSchema": "", "settingsDirective": "tb-polar-area-chart-widget-settings", diff --git a/application/src/main/data/json/system/widget_types/radar.json b/application/src/main/data/json/system/widget_types/radar.json index 07b0fa6981..a6961feb6d 100644 --- a/application/src/main/data/json/system/widget_types/radar.json +++ b/application/src/main/data/json/system/widget_types/radar.json @@ -9,9 +9,9 @@ "sizeX": 5, "sizeY": 4, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.radarChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.radarChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.radarChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.radarChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n embedActionsPanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n", "settingsSchema": "", "dataKeySettingsSchema": "", "settingsDirective": "tb-radar-chart-widget-settings", diff --git a/application/src/main/data/json/system/widget_types/radial_gauge.json b/application/src/main/data/json/system/widget_types/radial_gauge.json index ce09d31988..b463994347 100644 --- a/application/src/main/data/json/system/widget_types/radial_gauge.json +++ b/application/src/main/data/json/system/widget_types/radial_gauge.json @@ -17,7 +17,7 @@ "settingsDirective": "tb-analogue-radial-gauge-widget-settings", "hasBasicMode": true, "basicModeDirective": "tb-radial-gauge-basic-config", - "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Temp\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.7282710489093589,\"funcBody\":\"var value = prevValue + Math.random() * 50 - 25;\\nif (value < -100) {\\n\\tvalue = -100;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":false,\"backgroundColor\":\"rgb(255, 255, 255)\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{\"maxValue\":100,\"startAngle\":45,\"ticksAngle\":270,\"showBorder\":true,\"defaultColor\":\"#e65100\",\"needleCircleSize\":10,\"highlights\":[],\"showUnitTitle\":true,\"colorPlate\":\"#fff\",\"colorMajorTicks\":\"#444\",\"colorMinorTicks\":\"#666\",\"minorTicks\":10,\"valueInt\":3,\"valueDec\":0,\"highlightsWidth\":15,\"valueBox\":true,\"animation\":true,\"animationDuration\":500,\"animationRule\":\"cycle\",\"colorNeedleShadowUp\":\"rgba(2, 255, 255, 0)\",\"numbersFont\":{\"family\":\"Roboto\",\"size\":18,\"style\":\"normal\",\"weight\":\"500\",\"color\":\"#616161\"},\"titleFont\":{\"family\":\"Roboto\",\"size\":24,\"style\":\"normal\",\"weight\":\"500\",\"color\":\"#888\"},\"unitsFont\":{\"family\":\"Roboto\",\"size\":22,\"style\":\"normal\",\"weight\":\"500\",\"color\":\"#616161\"},\"valueFont\":{\"family\":\"Segment7Standard\",\"size\":36,\"style\":\"normal\",\"weight\":\"normal\",\"shadowColor\":\"rgba(0, 0, 0, 0.49)\",\"color\":\"#444\"},\"minValue\":-100,\"colorNeedleShadowDown\":\"rgba(188,143,143,0.45)\",\"colorValueBoxRect\":\"#888\",\"colorValueBoxRectEnd\":\"#666\",\"colorValueBoxBackground\":\"#babab2\",\"colorValueBoxShadow\":\"rgba(0,0,0,1)\"},\"title\":\"Radial gauge\",\"dropShadow\":true,\"enableFullscreen\":true,\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"widgetStyle\":{},\"widgetCss\":\"\",\"pageSize\":1024,\"decimals\":0,\"noDataDisplayMessage\":\"\",\"configMode\":\"basic\"}" + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Temp\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.7282710489093589,\"funcBody\":\"var value = prevValue + Math.random() * 50 - 25;\\nif (value < -100) {\\n\\tvalue = -100;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":false,\"backgroundColor\":\"rgb(255, 255, 255)\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{\"maxValue\":100,\"startAngle\":45,\"ticksAngle\":270,\"showBorder\":true,\"defaultColor\":\"#e65100\",\"needleCircleSize\":10,\"highlights\":[],\"showUnitTitle\":true,\"colorPlate\":\"#fff\",\"colorMajorTicks\":\"#444\",\"colorMinorTicks\":\"#666\",\"minorTicks\":10,\"valueInt\":3,\"highlightsWidth\":15,\"valueBox\":true,\"animation\":true,\"animationDuration\":500,\"animationRule\":\"cycle\",\"colorNeedleShadowUp\":\"rgba(2, 255, 255, 0)\",\"numbersFont\":{\"family\":\"Roboto\",\"size\":18,\"style\":\"normal\",\"weight\":\"500\",\"color\":\"#616161\"},\"titleFont\":{\"family\":\"Roboto\",\"size\":24,\"style\":\"normal\",\"weight\":\"500\",\"color\":\"#888\"},\"unitsFont\":{\"family\":\"Roboto\",\"size\":22,\"style\":\"normal\",\"weight\":\"500\",\"color\":\"#616161\"},\"valueFont\":{\"family\":\"Segment7Standard\",\"size\":36,\"style\":\"normal\",\"weight\":\"normal\",\"shadowColor\":\"rgba(0, 0, 0, 0.49)\",\"color\":\"#444\"},\"minValue\":-100,\"colorNeedleShadowDown\":\"rgba(188,143,143,0.45)\",\"colorValueBoxRect\":\"#888\",\"colorValueBoxRectEnd\":\"#666\",\"colorValueBoxBackground\":\"#babab2\",\"colorValueBoxShadow\":\"rgba(0,0,0,1)\"},\"title\":\"Radial gauge\",\"dropShadow\":true,\"enableFullscreen\":true,\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"widgetStyle\":{},\"widgetCss\":\"\",\"pageSize\":1024,\"decimals\":0,\"noDataDisplayMessage\":\"\",\"configMode\":\"basic\"}" }, "resources": [ { diff --git a/application/src/main/data/json/system/widget_types/range_chart.json b/application/src/main/data/json/system/widget_types/range_chart.json index ea5109873b..911d91876f 100644 --- a/application/src/main/data/json/system/widget_types/range_chart.json +++ b/application/src/main/data/json/system/widget_types/range_chart.json @@ -9,9 +9,9 @@ "sizeX": 8, "sizeY": 5, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": ".legend {\n font-size: 13px;\n line-height: 10px;\n}\n\n.legend table { \n border-spacing: 0px;\n border-collapse: separate;\n}\n\n.mouse-events .flot-overlay {\n cursor: crosshair; \n}\n\n", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.rangeChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.rangeChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries' }];\n }\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.rangeChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.rangeChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries' }];\n }\n };\n}\n", "settingsSchema": "{}", "dataKeySettingsSchema": "{}", "latestDataKeySettingsSchema": "{}", diff --git a/application/src/main/data/json/system/widget_types/route_map.json b/application/src/main/data/json/system/widget_types/route_map.json index f1d2cd571c..37346f1157 100644 --- a/application/src/main/data/json/system/widget_types/route_map.json +++ b/application/src/main/data/json/system/widget_types/route_map.json @@ -9,9 +9,9 @@ "sizeX": 8.5, "sizeY": 6, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n trip: true,\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n trip: true,\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}", "settingsForm": [], "dataKeySettingsForm": [], "latestDataKeySettingsForm": [], diff --git a/application/src/main/data/json/system/widget_types/state_chart.json b/application/src/main/data/json/system/widget_types/state_chart.json index ea65f55b43..d2a3031318 100644 --- a/application/src/main/data/json/system/widget_types/state_chart.json +++ b/application/src/main/data/json/system/widget_types/state_chart.json @@ -9,9 +9,9 @@ "sizeX": 8, "sizeY": 5, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n stateData: true,\n chartType: 'state',\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('state'),\n defaultDataKeysFunction: function() {\n return [{ name: 'state', label: 'State', type: 'timeseries', units: '', decimals: 0 }];\n }\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n stateData: true,\n chartType: 'state',\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings('state'),\n defaultDataKeysFunction: function() {\n return [{ name: 'state', label: 'State', type: 'timeseries', units: '', decimals: 0 }];\n }\n };\n}\n", "settingsSchema": "{}", "dataKeySettingsSchema": "{}", "latestDataKeySettingsSchema": "{}", diff --git a/application/src/main/data/json/system/widget_types/time_series_chart.json b/application/src/main/data/json/system/widget_types/time_series_chart.json index 449fffd11b..d61472b6f9 100644 --- a/application/src/main/data/json/system/widget_types/time_series_chart.json +++ b/application/src/main/data/json/system/widget_types/time_series_chart.json @@ -9,9 +9,9 @@ "sizeX": 8, "sizeY": 5, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings(),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.timeSeriesChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.timeSeriesChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n hasAdditionalLatestDataKeys: true,\n dataKeySettingsFunction: TbTimeSeriesChart.dataKeySettings(),\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries', units: '°C', decimals: 0 }];\n }\n };\n}\n", "settingsSchema": "{}", "dataKeySettingsSchema": "{}", "latestDataKeySettingsSchema": "{}", diff --git a/application/src/main/data/json/system/widget_types/trip_map.json b/application/src/main/data/json/system/widget_types/trip_map.json index a596e74c1f..8f54f64b6b 100644 --- a/application/src/main/data/json/system/widget_types/trip_map.json +++ b/application/src/main/data/json/system/widget_types/trip_map.json @@ -9,9 +9,9 @@ "sizeX": 8.5, "sizeY": 6, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n trip: true,\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.mapWidget.onInit();\n};\n\nself.typeParameters = function() {\n return {\n trip: true,\n hideDataTab: true,\n hideDataSettings: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n datasourcesOptional: true,\n additionalWidgetActionTypes: ['placeMapItem']\n };\n}", "settingsForm": [], "dataKeySettingsForm": [], "latestDataKeySettingsForm": [], diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 29c7a084f4..e91fbb823c 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -83,3 +83,13 @@ SET profile_data = profile_data WHERE profile_data->'configuration'->>'maxCalculatedFieldsPerEntity' IS NULL; -- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS END + +-- UPDATE TENANT PROFILE DEBUG DURATION START + +UPDATE tenant_profile +SET profile_data = jsonb_set(profile_data, '{configuration,maxDebugModeDurationMinutes}', '15', true) +WHERE + profile_data->'configuration' ? 'maxDebugModeDurationMinutes' = false + OR (profile_data->'configuration'->>'maxDebugModeDurationMinutes')::int = 0; + +-- UPDATE TENANT PROFILE DEBUG DURATION END diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index 350a5776cf..2959bfc8eb 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -21,6 +21,7 @@ import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorStopReason; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; @@ -47,6 +48,12 @@ public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { } } + @Override + public void destroy(TbActorStopReason stopReason, Throwable cause) throws TbActorException { + log.debug("[{}] Stopping CF entity actor.", processor.tenantId); + processor.stop(); + } + @Override protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { switch (msg.getMsgType()) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index a185b71d56..db65248a8f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -92,6 +92,12 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM this.ctx = ctx; } + public void stop() { + log.info("[{}][{}] Stopping entity actor.", tenantId, entityId); + states.clear(); + ctx.stop(ctx.getSelf()); + } + public void process(CalculatedFieldPartitionChangeMsg msg) { if (!systemContext.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, tenantId, entityId).isMyPartition()) { log.info("[{}] Stopping entity actor due to change partition event.", entityId); @@ -296,6 +302,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); } } + } else { + callback.onSuccess(); } } catch (Exception e) { throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); @@ -414,7 +422,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private Map mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, List removedTelemetryKeys) { Map deletedArguments = ctx.getArguments().entrySet().stream() - .filter(entry -> removedTelemetryKeys.contains(entry.getKey())) + .filter(entry -> removedTelemetryKeys.contains(entry.getValue().getRefEntityKey().getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index a5c935e83f..70ed2849e8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -20,6 +20,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorException; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorStopReason; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; @@ -52,6 +53,12 @@ public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor { } } + @Override + public void destroy(TbActorStopReason stopReason, Throwable cause) throws TbActorException { + log.debug("[{}] Stopping CF manager actor.", processor.tenantId); + processor.stop(); + } + @Override protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { switch (msg.getMsgType()) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index fc48e9ad3e..e109e83e4d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -56,7 +56,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; - /** * @author Andrew Shvayka */ @@ -92,8 +91,17 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware this.ctx = ctx; } + public void stop() { + log.info("[{}] Stopping CF manager actor.", tenantId); + calculatedFields.values().forEach(CalculatedFieldCtx::stop); + calculatedFields.clear(); + entityIdCalculatedFields.clear(); + entityIdCalculatedFieldLinks.clear(); + ctx.stop(ctx.getSelf()); + } + public void onFieldInitMsg(CalculatedFieldInitMsg msg) throws CalculatedFieldException { - log.info("[{}] Processing CF init message.", msg.getCf().getId()); + log.debug("[{}] Processing CF init message.", msg.getCf().getId()); var cf = msg.getCf(); var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { @@ -109,7 +117,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { - log.info("[{}] Processing CF link init message for entity [{}].", msg.getLink().getCalculatedFieldId(), msg.getLink().getEntityId()); + log.debug("[{}] Processing CF link init message for entity [{}].", msg.getLink().getCalculatedFieldId(), msg.getLink().getEntityId()); var link = msg.getLink(); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) @@ -122,7 +130,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var calculatedField = calculatedFields.get(cfId); if (calculatedField != null) { - msg.getState().setRequiredArguments(calculatedField.getArgNames()); + if (msg.getState() != null) { + msg.getState().setRequiredArguments(calculatedField.getArgNames()); + } log.debug("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); getOrCreateActor(msg.getId().entityId()).tell(msg); } else { @@ -131,7 +141,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { - log.info("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); + log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); var event = msg.getData().getEvent(); switch (entityType) { @@ -357,6 +367,10 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.debug("Received linked telemetry msg from entity [{}]", sourceEntityId); var proto = msg.getProto(); var linksList = proto.getLinksList(); + if (linksList.isEmpty()) { + log.debug("[{}] No CF links to process new telemetry.", msg.getTenantId()); + msg.getCallback().onSuccess(); + } for (var linkProto : linksList) { var link = fromProto(linkProto); var targetEntityId = link.entityId(); diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 846bde508d..728f715af4 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -28,6 +28,7 @@ import org.thingsboard.server.actors.TbEntityActorId; import org.thingsboard.server.actors.TbEntityTypeActorIdPredicate; import org.thingsboard.server.actors.TbStringActorId; import org.thingsboard.server.actors.calculatedField.CalculatedFieldManagerActorCreator; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; import org.thingsboard.server.actors.device.DeviceActorCreator; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; @@ -126,6 +127,9 @@ public class TenantActor extends RuleChainManagerActor { @Override public void destroy(TbActorStopReason stopReason, Throwable cause) { log.info("[{}] Stopping tenant actor.", tenantId); + if (cfActor != null) { + ctx.stop(cfActor.getActorId()); + } } @Override @@ -190,7 +194,11 @@ public class TenantActor extends RuleChainManagerActor { private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { if (cfActor == null) { - log.warn("[{}] CF Actor is not initialized.", tenantId); + if (msg instanceof CalculatedFieldStateRestoreMsg) { + log.warn("[{}] CF Actor is not initialized. ToCalculatedFieldSystemMsg: [{}]", tenantId, msg); + } else { + log.debug("[{}] CF Actor is not initialized. ToCalculatedFieldSystemMsg: [{}]", tenantId, msg); + } return; } if (priority) { diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 2ce4ade587..871798ccd1 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -82,7 +82,7 @@ public class ThingsboardSecurityConfiguration { public static final String TOKEN_BASED_AUTH_ENTRY_POINT = "/api/**"; public static final String WS_ENTRY_POINT = "/api/ws/**"; public static final String MAIL_OAUTH2_PROCESSING_ENTRY_POINT = "/api/admin/mail/oauth2/code"; - public static final String DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT = "/api/device-connectivity/mqtts/certificate/download"; + public static final String DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT = "/api/device-connectivity/*/certificate/download"; @Value("${server.http.max_payload_size:/api/image*/**=52428800;/api/resource/**=52428800;/api/**=16777216}") private String maxPayloadSizeConfig; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 0c4352dcea..fdbbb37568 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -122,6 +122,15 @@ public class CalculatedFieldCtx { } } + public void stop() { + if (calculatedFieldScriptEngine != null) { + calculatedFieldScriptEngine.destroy(); + } + if (customExpression != null) { + customExpression.remove(); + } + } + private CalculatedFieldScriptEngine initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { if (tbelInvokeService == null) { throw new IllegalArgumentException("TBEL script engine is disabled!"); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 90d4056afc..3620ad3639 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -97,7 +97,10 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta .scheduler(eventConsumer.getScheduler()) .taskExecutor(eventConsumer.getTaskExecutor()) .build(); - super.stateService = new KafkaQueueStateService<>(eventConsumer, stateConsumer); + super.stateService = KafkaQueueStateService., TbProtoQueueMsg>builder() + .eventConsumer(eventConsumer) + .stateConsumer(stateConsumer) + .build(); this.stateProducer = (TbKafkaProducerTemplate>) queueFactory.createCalculatedFieldStateProducer(); } diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java index 51c963ed2f..c7e17b62ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java @@ -76,8 +76,9 @@ public class DefaultEdqsApiService implements EdqsApiService { requestMsg.setCustomerIdLSB(customerId.getId().getLeastSignificantBits()); } - Integer partition = edqsPartitionService.resolvePartition(tenantId); - ListenableFuture> resultFuture = requestTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), requestMsg.build()), partition); + UUID key = UUID.randomUUID(); + Integer partition = edqsPartitionService.resolvePartition(tenantId, key); + ListenableFuture> resultFuture = requestTemplate.send(new TbProtoQueueMsg<>(key, requestMsg.build()), partition); return Futures.transform(resultFuture, msg -> { TransportProtos.EdqsResponseMsg responseMsg = msg.getValue().getResponseMsg(); return JacksonUtil.fromString(responseMsg.getValue(), EdqsResponse.class); diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index e823dee4e7..cd46c4ba96 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -59,7 +59,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.discovery.HashPartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.environment.DistributedLock; import org.thingsboard.server.queue.environment.DistributedLockService; import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; @@ -96,10 +95,8 @@ public class DefaultEdqsService implements EdqsService { private void init() { executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass()); eventsProducer = EdqsProducer.builder() - .queue(EdqsQueue.EVENTS) + .producer(queueFactory.createEdqsEventsProducer()) .partitionService(edqsPartitionService) - .topicService(topicService) - .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS)) .build(); syncLock = distributedLockService.getLock("edqs_sync"); } @@ -150,9 +147,12 @@ public class DefaultEdqsService implements EdqsService { syncLock.lock(); try { EdqsSyncState syncState = getSyncState(); - if (syncState != null && syncState.getStatus() == EdqsSyncStatus.FINISHED) { - log.info("EDQS sync is already finished"); - return; + if (syncState != null) { + EdqsSyncStatus status = syncState.getStatus(); + if (status == EdqsSyncStatus.FINISHED || status == EdqsSyncStatus.FAILED) { + log.info("EDQS sync is already " + status + ", ignoring the msg"); + return; + } } saveSyncState(EdqsSyncStatus.STARTED); diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java index 4ef552521b..239fd9dc42 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -17,11 +17,14 @@ package org.thingsboard.server.service.edqs; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import java.util.Collections; +import java.util.stream.Collectors; +import java.util.stream.IntStream; @Service @ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") @@ -29,9 +32,14 @@ public class KafkaEdqsSyncService extends EdqsSyncService { private final boolean syncNeeded; - public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings) { + public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings, EdqsConfig edqsConfig) { TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap()); - this.syncNeeded = kafkaAdmin.isTopicEmpty(EdqsQueue.EVENTS.getTopic()); + this.syncNeeded = kafkaAdmin.areAllTopicsEmpty(IntStream.range(0, edqsConfig.getPartitions()) + .mapToObj(partition -> TopicPartitionInfo.builder() + .topic(edqsConfig.getEventsTopic()) + .partition(partition) + .build().getFullTopicName()) + .collect(Collectors.toSet())); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 870ce3838b..e9ef8c5ace 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -122,6 +122,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.thingsboard.common.util.DebugModeUtil.DEBUG_MODE_DEFAULT_DURATION_MINUTES; import static org.thingsboard.server.common.data.DataConstants.DEFAULT_DEVICE_TYPE; import static org.thingsboard.server.service.security.auth.jwt.settings.DefaultJwtSettingsService.isSigningKeyDefault; import static org.thingsboard.server.service.security.auth.jwt.settings.DefaultJwtSettingsService.validateKeyLength; @@ -199,7 +200,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { tenantProfileService.findOrCreateDefaultTenantProfile(TenantId.SYS_TENANT_ID); TenantProfileData isolatedRuleEngineTenantProfileData = new TenantProfileData(); - isolatedRuleEngineTenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + DefaultTenantProfileConfiguration configuration = new DefaultTenantProfileConfiguration(); + configuration.setMaxDebugModeDurationMinutes(DEBUG_MODE_DEFAULT_DURATION_MINUTES); + isolatedRuleEngineTenantProfileData.setConfiguration(configuration); TenantProfileQueueConfiguration mainQueueConfiguration = new TenantProfileQueueConfiguration(); mainQueueConfiguration.setName(DataConstants.MAIN_QUEUE_NAME); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index ac62576714..125a06299d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.queue; -import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -55,7 +54,7 @@ import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; -import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.queue.processing.AbstractPartitionBasedConsumerService; import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; @@ -72,7 +71,7 @@ import java.util.stream.Collectors; @Service @TbRuleEngineComponent @Slf4j -public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService implements TbCalculatedFieldConsumerService { +public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBasedConsumerService implements TbCalculatedFieldConsumerService { @Value("${queue.calculated_fields.poll_interval:25}") private long pollInterval; @@ -99,10 +98,8 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer this.stateService = stateService; } - @PostConstruct - public void init() { - super.init("tb-cf"); - + @Override + protected void onStartUp() { var queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); PartitionedQueueConsumerManager> eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(queueKey) @@ -129,7 +126,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer } @Override - protected void onTbApplicationEvent(PartitionChangeEvent event) { + protected void onPartitionChangeEvent(PartitionChangeEvent event) { try { event.getNewPartitions().forEach((queueKey, partitions) -> { if (queueKey.getQueueName().equals(DataConstants.CF_QUEUE_NAME)) { @@ -193,6 +190,11 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer return ServiceType.TB_RULE_ENGINE; } + @Override + protected String getPrefix() { + return "tb-cf"; + } + @Override protected long getNotificationPollDuration() { return pollInterval; @@ -210,7 +212,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected TbQueueConsumer> createNotificationsConsumer() { - return queueFactory.createToCalculatedFieldNotificationsMsgConsumer(); + return queueFactory.createToCalculatedFieldNotificationMsgConsumer(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 9cc743e510..98f857750a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.queue; -import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; @@ -50,7 +49,7 @@ import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; -import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.queue.processing.AbstractPartitionBasedConsumerService; import org.thingsboard.server.service.queue.ruleengine.TbRuleEngineConsumerContext; import org.thingsboard.server.service.queue.ruleengine.TbRuleEngineQueueConsumerManager; import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; @@ -67,7 +66,7 @@ import java.util.stream.Collectors; @Service @TbRuleEngineComponent @Slf4j -public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { +public class DefaultTbRuleEngineConsumerService extends AbstractPartitionBasedConsumerService implements TbRuleEngineConsumerService { private final TbRuleEngineConsumerContext ctx; private final QueueService queueService; @@ -93,9 +92,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< this.queueService = queueService; } - @PostConstruct - public void init() { - super.init("tb-rule-engine"); + @Override + protected void onStartUp() { List queues = queueService.findAllQueues(); for (Queue configuration : queues) { if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { @@ -106,7 +104,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } @Override - protected void onTbApplicationEvent(PartitionChangeEvent event) { + protected void onPartitionChangeEvent(PartitionChangeEvent event) { event.getNewPartitions().forEach((queueKey, partitions) -> { if (DataConstants.CF_QUEUE_NAME.equals(queueKey.getQueueName()) || DataConstants.CF_STATES_QUEUE_NAME.equals(queueKey.getQueueName())) { return; @@ -150,6 +148,11 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< return ServiceType.TB_RULE_ENGINE; } + @Override + protected String getPrefix() { + return "tb-rule-engine"; + } + @Override protected long getNotificationPollDuration() { return ctx.getPollDuration(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractPartitionBasedConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractPartitionBasedConsumerService.java new file mode 100644 index 0000000000..97aa81d41c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractPartitionBasedConsumerService.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import jakarta.annotation.PostConstruct; +import org.springframework.context.ApplicationEventPublisher; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; + +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +public abstract class AbstractPartitionBasedConsumerService extends AbstractConsumerService { + + private final Lock startupLock = new ReentrantLock(); + private volatile boolean started = false; + private PartitionChangeEvent lastPartitionChangeEvent; + + public AbstractPartitionBasedConsumerService(ActorSystemContext actorContext, + TbTenantProfileCache tenantProfileCache, + TbDeviceProfileCache deviceProfileCache, + TbAssetProfileCache assetProfileCache, + CalculatedFieldCache calculatedFieldCache, + TbApiUsageStateService apiUsageStateService, + PartitionService partitionService, + ApplicationEventPublisher eventPublisher, + JwtSettingsService jwtSettingsService) { + super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); + } + + @PostConstruct + public void init() { + super.init(getPrefix()); + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + @Override + public void afterStartUp() { + super.afterStartUp(); + onStartUp(); + startupLock.lock(); + try { + onPartitionChangeEvent(lastPartitionChangeEvent); + started = true; + } finally { + startupLock.unlock(); + } + } + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + if (!started) { + startupLock.lock(); + try { + if (!started) { + lastPartitionChangeEvent = event; + return; + } + } finally { + startupLock.unlock(); + } + } + onPartitionChangeEvent(event); + } + + protected abstract void onStartUp(); + + protected abstract void onPartitionChangeEvent(PartitionChangeEvent event); + + protected abstract String getPrefix(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 9e9ca42e83..0f308d85ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -725,7 +725,12 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription()); ctx.setInitialDataSent(true); } else { - update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData(), ctx.getMaxEntitiesPerDataSubscription()); + // if ctx has timeseries subscription, timeseries values are cleared after each update and is empty in ctx data, + // so to avoid sending timeseries update with empty map we set it to null + List preparedData = ctx.getData().getData().stream() + .map(entityData -> new EntityData(entityData.getEntityId(), entityData.getLatest(), null)) + .toList(); + update = new EntityDataUpdate(ctx.getCmdId(), null, preparedData, ctx.getMaxEntitiesPerDataSubscription()); } ctx.sendWsMsg(update); } finally { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index c61a993d56..7ede65e375 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1645,12 +1645,12 @@ queue: calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Calculated Field State topics calculated-field-state: "${TB_QUEUE_KAFKA_CF_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:104857600000;partitions:1;min.insync.replicas:1;cleanup.policy:compact}" - # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions - edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" - # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions - edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" - # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions - edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" + # Kafka properties for EDQS events topics + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:86400000;segment.bytes:52428800;retention.bytes:-1;partitions:1;min.insync.replicas:1}" + # Kafka properties for EDQS requests topic (default: 3 minutes retention) + edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for EDQS state topic (infinite retention, compaction) + edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:1;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1748,6 +1748,10 @@ queue: partitions: "${TB_EDQS_PARTITIONS:12}" # EDQS partitioning strategy: tenant (partition is resolved by tenant id) or none (no specific strategy, resolving by message key) partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" + # EDQS events topic + events_topic: "${TB_EDQS_EVENTS_TOPIC:edqs.events}" + # EDQS state topic + state_topic: "${TB_EDQS_STATE_TOPIC:edqs.state}" # EDQS requests topic requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" # EDQS responses topic diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 011399e883..90b7ea7c7b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -17,8 +17,6 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; -import org.awaitility.Awaitility; -import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -73,7 +71,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.function.BiPredicate; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index 50c80d08c7..5264e69bd3 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -110,7 +110,7 @@ public class EdqsEntityServiceTest extends EntityServiceTest { @Override protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { - return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries), + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> loadAllData(query, expectedTelemetries.size()), loadedEntities -> loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList().containsAll(expectedTelemetries)); } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 1e7f5384b5..e5c4009715 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -1698,6 +1698,19 @@ public class EntityServiceTest extends AbstractControllerTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); findByQueryAndCheckTelemetry(query, EntityKeyType.TIME_SERIES, "temperature", deviceHighTemperatures); + // change sort order to sort by temperature + temperatures.sort(Comparator.naturalOrder()); + List expectedSortedList = temperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList()); + + EntityDataSortOrder sortByTempOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "temperature"), EntityDataSortOrder.Direction.ASC); + EntityDataPageLink sortByTempPageLink = new EntityDataPageLink(10, 0, null, sortByTempOrder); + EntityDataQuery querySortByTemp = new EntityDataQuery(filter, sortByTempPageLink, entityFields, latestValues, null); + + List loadedEntities = loadAllData(querySortByTemp, deviceTemperatures.size()); + List entitiesTelemetry = loadedEntities.stream().map(entityData -> entityData.getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).toList(); + assertThat(entitiesTelemetry).containsExactlyElementsOf(expectedSortedList); + deviceService.deleteDevicesByTenantId(tenantId); } @@ -2377,14 +2390,14 @@ public class EntityServiceTest extends AbstractControllerTest { } protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetry) { - List loadedEntities = findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetry); + List loadedEntities = loadAllData(query, expectedTelemetry.size()); List entitiesTelemetry = loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList(); assertThat(entitiesTelemetry).containsExactlyInAnyOrderElementsOf(expectedTelemetry); return loadedEntities; } - protected List findEntitiesTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { - PageData data = findByQueryAndCheck(query, expectedTelemetries.size()); + protected List loadAllData(EntityDataQuery query, int expectedSize) { + PageData data = findByQueryAndCheck(query, expectedSize); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java index a6f30c8004..75829dfbbf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java @@ -17,7 +17,7 @@ package org.thingsboard.server.common.data.edqs; import org.thingsboard.server.common.data.kv.DataType; -public interface DataPoint { +public interface DataPoint extends Comparable { String NOT_SUPPORTED = "Not supported!"; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java index bf2a3f6da7..9ce300df6a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java @@ -50,10 +50,10 @@ public class CustomerData extends BaseEntityData { entitiesById.computeIfAbsent(ed.getEntityType(), et -> new ConcurrentHashMap<>()).put(ed.getId(), ed); } - public boolean remove(EntityData> ed) { - var map = entitiesById.get(ed.getEntityType()); + public boolean remove(EntityType entityType, UUID entityId) { + var map = entitiesById.get(entityType); if (map != null) { - return map.remove(ed.getId()) != null; + return map.remove(entityId) != null; } else { return false; } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java index fd2d099281..1fdb731af6 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java @@ -17,6 +17,7 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.common.data.edqs.DataPoint; @RequiredArgsConstructor @@ -54,4 +55,9 @@ public abstract class AbstractDataPoint implements DataPoint { return valueToString(); } + @Override + public int compareTo(DataPoint dataPoint) { + return StringUtils.compareIgnoreCase(valueToString(), dataPoint.valueToString()); + } + } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java index 83d91d8f75..70a14917ba 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java @@ -16,6 +16,7 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; +import org.thingsboard.server.common.data.edqs.DataPoint; import org.thingsboard.server.common.data.kv.DataType; public class BoolDataPoint extends AbstractDataPoint { @@ -43,4 +44,8 @@ public class BoolDataPoint extends AbstractDataPoint { return Boolean.toString(value); } + @Override + public int compareTo(DataPoint dataPoint) { + return Boolean.compare(value, dataPoint.getBool()); + } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java index 21b355bc46..2ca8f2c03a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java @@ -16,6 +16,7 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; +import org.thingsboard.server.common.data.edqs.DataPoint; import org.thingsboard.server.common.data.kv.DataType; public class DoubleDataPoint extends AbstractDataPoint { @@ -43,4 +44,8 @@ public class DoubleDataPoint extends AbstractDataPoint { return Double.toString(value); } + @Override + public int compareTo(DataPoint dataPoint) { + return Double.compare(value, dataPoint.getDouble()); + } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java index 7fbe90e814..92c0a972e7 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java @@ -16,6 +16,7 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; +import org.thingsboard.server.common.data.edqs.DataPoint; import org.thingsboard.server.common.data.kv.DataType; public class LongDataPoint extends AbstractDataPoint { @@ -47,4 +48,9 @@ public class LongDataPoint extends AbstractDataPoint { public String valueToString() { return Long.toString(value); } + + @Override + public int compareTo(DataPoint dataPoint) { + return Long.compare(value, dataPoint.getLong()); + } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 7ddc9147df..07575220eb 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -62,7 +62,6 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.edqs.EdqsComponent; import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy; -import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.edqs.EdqsQueueFactory; import org.thingsboard.server.queue.util.AfterStartUp; @@ -123,8 +122,8 @@ public class EdqsProcessor implements TbQueueHandler, }; eventConsumer = PartitionedQueueConsumerManager.>create() - .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.EVENTS.getTopic())) - .topic(EdqsQueue.EVENTS.getTopic()) + .queueKey(new QueueKey(ServiceType.EDQS, config.getEventsTopic())) + .topic(config.getEventsTopic()) .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer, config) -> { for (TbProtoQueueMsg queueMsg : msgs) { @@ -133,14 +132,14 @@ public class EdqsProcessor implements TbQueueHandler, } try { ToEdqsMsg msg = queueMsg.getValue(); - process(msg, EdqsQueue.EVENTS); + process(msg, true); } catch (Exception t) { log.error("Failed to process message: {}", queueMsg, t); } } consumer.commit(); }) - .consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS)) + .consumerCreator((config, partitionId) -> queueFactory.createEdqsEventsConsumer()) .queueAdmin(queueFactory.getEdqsQueueAdmin()) .consumerExecutor(consumersExecutor) .taskExecutor(taskExecutor) @@ -164,25 +163,27 @@ public class EdqsProcessor implements TbQueueHandler, } try { Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); - Set partitions = newPartitions.stream() - .map(tpi -> tpi.withUseInternalPartition(true)) - .collect(Collectors.toSet()); - stateService.process(withTopic(partitions, EdqsQueue.STATE.getTopic())); + stateService.process(withTopic(newPartitions, config.getStateTopic())); // eventsConsumer's partitions are updated by stateService - responseTemplate.subscribe(withTopic(partitions, config.getRequestsTopic())); // FIXME: we subscribe to partitions before we are ready. implement consumer-per-partition version for request template + responseTemplate.subscribe(withTopic(newPartitions, config.getRequestsTopic())); // TODO: we subscribe to partitions before we are ready. implement consumer-per-partition version for request template Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); if (CollectionsUtil.isNotEmpty(oldPartitions)) { Set removedPartitions = Sets.difference(oldPartitions, newPartitions).stream() .map(tpi -> tpi.getPartition().orElse(-1)).collect(Collectors.toSet()); - if (config.getPartitioningStrategy() != EdqsPartitioningStrategy.TENANT && !removedPartitions.isEmpty()) { + if (removedPartitions.isEmpty()) { + return; + } + + if (config.getPartitioningStrategy() == EdqsPartitioningStrategy.TENANT) { + repository.clearIf(tenantId -> { + Integer partition = partitionService.resolvePartition(tenantId, null); + return removedPartitions.contains(partition); + }); + } else { log.warn("Partitions {} were removed but shouldn't be (due to NONE partitioning strategy)", removedPartitions); } - repository.clearIf(tenantId -> { - Integer partition = partitionService.resolvePartition(tenantId); - return partition != null && removedPartitions.contains(partition); - }); } } catch (Throwable t) { log.error("Failed to handle partition change event {}", event, t); @@ -233,7 +234,7 @@ public class EdqsProcessor implements TbQueueHandler, return response; } - public void process(ToEdqsMsg edqsMsg, EdqsQueue queue) { + public void process(ToEdqsMsg edqsMsg, boolean backup) { log.trace("Processing message: {}", edqsMsg); if (edqsMsg.hasEventMsg()) { EdqsEventMsg eventMsg = edqsMsg.getEventMsg(); @@ -250,7 +251,7 @@ public class EdqsProcessor implements TbQueueHandler, } else if (!ObjectType.unversionedTypes.contains(objectType)) { log.warn("[{}] {} {} doesn't have version", tenantId, objectType, key); } - if (queue != EdqsQueue.STATE) { + if (backup) { stateService.save(tenantId, objectType, key, eventType, edqsMsg); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java index be1f0481be..cc4f913d38 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java @@ -16,6 +16,7 @@ package org.thingsboard.server.edqs.processor; import lombok.Builder; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.errors.RecordTooLargeException; import org.thingsboard.server.common.data.ObjectType; @@ -27,60 +28,41 @@ import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; @Slf4j +@Builder +@RequiredArgsConstructor public class EdqsProducer { - private final EdqsQueue queue; - private final EdqsPartitionService partitionService; - private final TopicService topicService; - private final TbQueueProducer> producer; - - @Builder - public EdqsProducer(EdqsQueue queue, - EdqsPartitionService partitionService, - TopicService topicService, - TbQueueProducer> producer) { - this.queue = queue; - this.partitionService = partitionService; - this.topicService = topicService; - this.producer = producer; - } + private final EdqsPartitionService partitionService; public void send(TenantId tenantId, ObjectType type, String key, ToEdqsMsg msg) { - String topic = topicService.buildTopicName(queue.getTopic()); + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(producer.getDefaultTopic()) + .partition(partitionService.resolvePartition(tenantId, key)) + .build(); TbQueueCallback callback = new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { - log.trace("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, topic, msg); + log.trace("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, tpi, msg); } @Override public void onFailure(Throwable t) { if (t instanceof RecordTooLargeException) { if (!log.isDebugEnabled()) { - log.warn("[{}][{}][{}] Failed to publish msg to {}", tenantId, type, key, topic, t); // not logging the whole message + log.warn("[{}][{}][{}] Failed to publish msg to {}", tenantId, type, key, tpi, t); // not logging the whole message return; } } - log.warn("[{}][{}][{}] Failed to publish msg to {}: {}", tenantId, type, key, topic, msg, t); + log.warn("[{}][{}][{}] Failed to publish msg to {}: {}", tenantId, type, key, tpi, msg, t); } }; if (producer instanceof TbKafkaProducerTemplate> kafkaProducer) { - TopicPartitionInfo tpi = TopicPartitionInfo.builder() - .topic(topic) - .partition(partitionService.resolvePartition(tenantId)) - .useInternalPartition(true) - .build(); kafkaProducer.send(tpi, key, new TbProtoQueueMsg<>(null, msg), callback); // specifying custom key for compaction } else { - TopicPartitionInfo tpi = TopicPartitionInfo.builder() - .topic(topic) - .build(); producer.send(tpi, new TbProtoQueueMsg<>(null, msg), callback); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java index 026c470ce6..18936a2696 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java @@ -16,6 +16,7 @@ package org.thingsboard.server.edqs.query; import lombok.Data; +import org.thingsboard.server.common.data.edqs.DataPoint; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.edqs.data.EntityData; @@ -26,7 +27,7 @@ import java.util.UUID; public class SortableEntityData { private final EntityData entityData; - private String sortValue; + private DataPoint sortValue; public UUID getId(){ return entityData.getId(); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java index e4cded3e3e..0c83ae4e61 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.edqs.query.processor; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.DataPoint; import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.edqs.data.EntityData; @@ -50,7 +51,7 @@ public abstract class AbstractQueryProcessor implements protected SortableEntityData toSortData(EntityData> ed) { SortableEntityData sortData = new SortableEntityData(ed); - sortData.setSortValue(getSortValue(ed, sortKey)); + sortData.setSortValue(getSortValue(ed, sortKey, ctx)); return sortData; } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java index 8ee7338a4f..2842d57ff0 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Queue; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; public abstract class AbstractRelationQueryProcessor extends AbstractQueryProcessor { @@ -89,7 +90,7 @@ public abstract class AbstractRelationQueryProcessor ext private List processTenantQuery(Set> entities) { return entities.stream() .map(this::toSortData) - .toList(); + .collect(Collectors.toList()); } private List processCustomerQuery(Set> entities) { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 870574a786..b9f3589280 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -150,8 +150,9 @@ public class TenantRepo { } } else if (RelationTypeGroup.DASHBOARD.equals(entity.getTypeGroup())) { if (EntityRelation.CONTAINS_TYPE.equals(entity.getType()) && entity.getFrom().getEntityType() == EntityType.CUSTOMER) { - ((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(entity.getFrom().getId(), CustomerData::new)) - .addOrUpdate(getEntityMap(EntityType.DASHBOARD).get(entity.getTo().getId())); + CustomerData customerData = (CustomerData) getOrCreate(entity.getFrom()); + EntityData> dashboardData = getOrCreate(entity.getTo()); + customerData.addOrUpdate(dashboardData); } } } finally { @@ -170,8 +171,10 @@ public class TenantRepo { } } else if (RelationTypeGroup.DASHBOARD.equals(entityRelation.getTypeGroup())) { if (EntityRelation.CONTAINS_TYPE.equals(entityRelation.getType()) && entityRelation.getFrom().getEntityType() == EntityType.CUSTOMER) { - ((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(entityRelation.getFrom().getId(), CustomerData::new)) - .remove(getEntityMap(EntityType.DASHBOARD).get(entityRelation.getTo().getId())); + CustomerData customerData = (CustomerData) get(entityRelation.getFrom()); + if (customerData != null) { + customerData.remove(EntityType.DASHBOARD, entityRelation.getTo().getId()); + } } } } @@ -197,13 +200,13 @@ public class TenantRepo { entityData.setCustomerId(newCustomerId); if (entityIdMismatch(oldCustomerId, newCustomerId)) { if (oldCustomerId != null) { - CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId); + CustomerData old = (CustomerData) get(EntityType.CUSTOMER, oldCustomerId); if (old != null) { - old.remove(entityData); + old.remove(entityType, entityId); } } if (newCustomerId != null) { - CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new); + CustomerData newData = (CustomerData) getOrCreate(EntityType.CUSTOMER, newCustomerId); newData.addOrUpdate(entityData); } } @@ -223,11 +226,12 @@ public class TenantRepo { getEntitySet(entityType).remove(removed); } edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); + UUID customerId = removed.getCustomerId(); if (customerId != null) { - CustomerData customerData = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(customerId); + CustomerData customerData = (CustomerData) get(EntityType.CUSTOMER, customerId); if (customerData != null) { - customerData.remove(removed); + customerData.remove(entityType, entityId); } } } @@ -303,7 +307,11 @@ public class TenantRepo { } private EntityData> get(EntityId entityId) { - return getEntityMap(entityId.getEntityType()).get(entityId.getId()); + return get(entityId.getEntityType(), entityId.getId()); + } + + private EntityData> get(EntityType entityType, UUID entityId) { + return getEntityMap(entityType).get(entityId); } private EntityData> constructEntityData(EntityType entityType, UUID id) { @@ -425,7 +433,7 @@ public class TenantRepo { EntityType entityType = entityId.getEntityType(); return switch (entityType) { case CUSTOMER, TENANT -> { - EntityFields fields = getEntityMap(entityType).get(entityId.getId()).getFields(); + EntityFields fields = get(entityId).getFields(); yield fields != null ? fields.getName() : ""; } default -> throw new RuntimeException("Unsupported entity type: " + entityType); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java index 94e9437650..e2fbf9a981 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java @@ -29,11 +29,14 @@ public class EdqsPartitionService { private final HashPartitionService hashPartitionService; private final EdqsConfig edqsConfig; - public Integer resolvePartition(TenantId tenantId) { + public Integer resolvePartition(TenantId tenantId, Object key) { if (edqsConfig.getPartitioningStrategy() == EdqsPartitioningStrategy.TENANT) { return hashPartitionService.resolvePartitionIndex(tenantId.getId(), edqsConfig.getPartitions()); } else { - return null; + if (key == null) { + throw new IllegalArgumentException("Partitioning key is missing but partitioning strategy is not TENANT"); + } + return hashPartitionService.resolvePartitionIndex(key.toString(), edqsConfig.getPartitions()); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index 85b8e92387..efdb1ead1c 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -36,15 +36,19 @@ import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; import org.thingsboard.server.queue.common.state.KafkaQueueStateService; import org.thingsboard.server.queue.common.state.QueueStateService; import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsConfig; -import org.thingsboard.server.queue.edqs.EdqsQueue; -import org.thingsboard.server.queue.edqs.EdqsQueueFactory; import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; +import org.thingsboard.server.queue.edqs.KafkaEdqsQueueFactory; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; @Service @RequiredArgsConstructor @@ -54,8 +58,7 @@ public class KafkaEdqsStateService implements EdqsStateService { private final EdqsConfig config; private final EdqsPartitionService partitionService; - private final EdqsQueueFactory queueFactory; - private final TopicService topicService; + private final KafkaEdqsQueueFactory queueFactory; @Autowired @Lazy private EdqsProcessor edqsProcessor; @@ -71,15 +74,16 @@ public class KafkaEdqsStateService implements EdqsStateService { @Override public void init(PartitionedQueueConsumerManager> eventConsumer) { + TbKafkaAdmin queueAdmin = queueFactory.getEdqsQueueAdmin(); stateConsumer = PartitionedQueueConsumerManager.>create() - .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.STATE.getTopic())) - .topic(EdqsQueue.STATE.getTopic()) + .queueKey(new QueueKey(ServiceType.EDQS, config.getStateTopic())) + .topic(config.getStateTopic()) .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer, config) -> { for (TbProtoQueueMsg queueMsg : msgs) { try { ToEdqsMsg msg = queueMsg.getValue(); - edqsProcessor.process(msg, EdqsQueue.STATE); + edqsProcessor.process(msg, false); if (stateReadCount.incrementAndGet() % 100000 == 0) { log.info("[state] Processed {} msgs", stateReadCount.get()); } @@ -89,15 +93,15 @@ public class KafkaEdqsStateService implements EdqsStateService { } consumer.commit(); }) - .consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.STATE)) - .queueAdmin(queueFactory.getEdqsQueueAdmin()) + .consumerCreator((config, partitionId) -> queueFactory.createEdqsStateConsumer()) + .queueAdmin(queueAdmin) .consumerExecutor(eventConsumer.getConsumerExecutor()) .taskExecutor(eventConsumer.getTaskExecutor()) .scheduler(eventConsumer.getScheduler()) .uncaughtErrorHandler(edqsProcessor.getErrorHandler()) .build(); - queueStateService = new KafkaQueueStateService<>(eventConsumer, stateConsumer); + TbKafkaConsumerTemplate> eventsToBackupKafkaConsumer = queueFactory.createEdqsEventsToBackupConsumer(); eventsToBackupConsumer = QueueConsumerManager.>builder() .name("edqs-events-to-backup-consumer") .pollInterval(config.getPollInterval()) @@ -135,23 +139,47 @@ public class KafkaEdqsStateService implements EdqsStateService { } consumer.commit(); }) - .consumerCreator(() -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS, "events-to-backup-consumer-group")) // shared by all instances consumer group + .consumerCreator(() -> eventsToBackupKafkaConsumer) .consumerExecutor(eventConsumer.getConsumerExecutor()) .threadPrefix("edqs-events-to-backup") .build(); stateProducer = EdqsProducer.builder() - .queue(EdqsQueue.STATE) + .producer(queueFactory.createEdqsStateProducer()) .partitionService(partitionService) - .topicService(topicService) - .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.STATE)) + .build(); + + queueStateService = KafkaQueueStateService., TbProtoQueueMsg>builder() + .eventConsumer(eventConsumer) + .stateConsumer(stateConsumer) + .eventsStartOffsetsProvider(() -> { + // taking start offsets for events topics from the events-to-backup consumer group, + // since eventConsumer doesn't use consumer group management and thus offset tracking + // (because we need to be able to consume the same topic-partition by multiple instances) + Map offsets = new HashMap<>(); + try { + queueAdmin.getConsumerGroupOffsets(eventsToBackupKafkaConsumer.getGroupId()) + .forEach((topicPartition, offsetAndMetadata) -> { + offsets.put(topicPartition.topic(), offsetAndMetadata.offset()); + }); + } catch (Exception e) { + log.error("Failed to get consumer group offsets for {}", eventsToBackupKafkaConsumer.getGroupId(), e); + } + return offsets; + }) .build(); } @Override public void process(Set partitions) { if (queueStateService.getPartitions().isEmpty()) { - eventsToBackupConsumer.subscribe(); + Set allPartitions = IntStream.range(0, config.getPartitions()) + .mapToObj(partition -> TopicPartitionInfo.builder() + .topic(config.getEventsTopic()) + .partition(partition) + .build()) + .collect(Collectors.toSet()); + eventsToBackupConsumer.subscribe(allPartitions); eventsToBackupConsumer.launch(); } queueStateService.update(new QueueKey(ServiceType.EDQS), partitions); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java index 383115ddf1..cde21edfaf 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -29,7 +29,6 @@ import org.thingsboard.server.edqs.util.EdqsRocksDb; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; -import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; import java.util.Set; @@ -61,14 +60,14 @@ public class LocalEdqsStateService implements EdqsStateService { try { ToEdqsMsg edqsMsg = ToEdqsMsg.parseFrom(value); log.trace("[{}] Restored msg from RocksDB: {}", key, edqsMsg); - processor.process(edqsMsg, EdqsQueue.STATE); + processor.process(edqsMsg, false); } catch (Exception e) { log.error("[{}] Failed to restore value", key, e); } }); log.info("Restore completed"); } - eventConsumer.update(withTopic(partitions, EdqsQueue.EVENTS.getTopic())); + eventConsumer.update(withTopic(partitions, eventConsumer.getTopic())); this.partitions = partitions; } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java index 3bf12752a0..71b58d759a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -67,10 +67,10 @@ import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.Co @Slf4j public class RepositoryUtils { - public static final Comparator SORT_ASC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse(""), String.CASE_INSENSITIVE_ORDER) + public static final Comparator SORT_ASC = Comparator.comparing(SortableEntityData::getSortValue, Comparator.nullsFirst(Comparator.naturalOrder())) .thenComparing(sp -> sp.getId().toString()); - public static final Comparator SORT_DESC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse(""), String.CASE_INSENSITIVE_ORDER) + public static final Comparator SORT_DESC = Comparator.comparing(SortableEntityData::getSortValue, Comparator.nullsFirst(Comparator.naturalOrder())) .thenComparing(sp -> sp.getId().toString()).reversed(); public static EntityType resolveEntityType(EntityFilter entityFilter) { @@ -348,24 +348,11 @@ public class RepositoryUtils { } } - public static String getSortValue(EntityData entity, DataKey sortKey) { + public static DataPoint getSortValue(EntityData entity, DataKey sortKey, QueryContext queryContext) { if (sortKey == null) { return null; } - switch (sortKey.type()) { - case ENTITY_FIELD -> { - return entity.getField(sortKey.key()); - } - case ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE, SERVER_ATTRIBUTE -> { - var dp = entity.getAttr(sortKey.keyId(), sortKey.type()); - return dp != null ? dp.valueToString() : ""; - } - case TIME_SERIES -> { - var dp = entity.getTs(sortKey.keyId()); - return dp != null ? dp.valueToString() : ""; - } - default -> throw new IllegalStateException("toSortKey is not implemented for type: " + sortKey.type()); - } + return entity.getDataPoint(sortKey, queryContext); } public static boolean checkFilters(EdqsQuery query, EntityData entity) { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index f09826e9b6..b18debaf49 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -90,9 +90,7 @@ public class TopicPartitionInfo { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; TopicPartitionInfo that = (TopicPartitionInfo) o; - return topic.equals(that.topic) && - Objects.equals(tenantId, that.tenantId) && - Objects.equals(partition, that.partition) && + return Objects.equals(partition, that.partition) && fullTopicName.equals(that.fullTopicName); } diff --git a/common/message/src/test/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfoTest.java b/common/message/src/test/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfoTest.java index ec1e106dd8..f222a5a094 100644 --- a/common/message/src/test/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfoTest.java +++ b/common/message/src/test/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfoTest.java @@ -18,12 +18,16 @@ package org.thingsboard.server.common.msg.queue; import org.junit.jupiter.api.Test; import org.thingsboard.server.common.data.id.TenantId; +import java.util.UUID; + import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; public class TopicPartitionInfoTest { + private final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + @Test public void givenTopicPartitionInfo_whenEquals_thenTrue() { @@ -52,13 +56,13 @@ public class TopicPartitionInfoTest { assertThat(TopicPartitionInfo.builder() .topic("tb_core") - .tenantId(TenantId.SYS_TENANT_ID) + .tenantId(tenantId) .partition(4) .myPartition(true) //will ignored on equals .build() , is(TopicPartitionInfo.builder() .topic("tb_core") - .tenantId(TenantId.SYS_TENANT_ID) + .tenantId(tenantId) .partition(4) .myPartition(true) //will ignored on equals .build())); @@ -109,7 +113,7 @@ public class TopicPartitionInfoTest { assertThat(TopicPartitionInfo.builder() .topic("tb_core") - .tenantId(TenantId.SYS_TENANT_ID) + .tenantId(tenantId) .partition(4) .myPartition(true) //will ignored on equals .build() @@ -117,7 +121,7 @@ public class TopicPartitionInfoTest { assertThat(TopicPartitionInfo.builder() .topic("tb_core") - .tenantId(TenantId.SYS_TENANT_ID) + .tenantId(tenantId) .partition(4) .myPartition(false) //will ignored on equals .build() diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 4efb297491..1beb505595 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -263,7 +263,6 @@ public class DefaultTbQueueRequestTemplate { @@ -296,7 +298,7 @@ public class MainQueueConsumerManager removedPartitions) { @@ -304,13 +306,19 @@ public class MainQueueConsumerManager Optional.ofNullable(consumers.remove(tpi)).ifPresent(TbQueueConsumerTask::awaitCompletion)); } - protected void addPartitions(Set partitions, Consumer onStop) { + protected void addPartitions(Set partitions, Consumer onStop, Function startOffsetProvider) { partitions.forEach(tpi -> { Integer partitionId = tpi.getPartition().orElse(-1); String key = queueKey + "-" + partitionId; Runnable callback = onStop != null ? () -> onStop.accept(tpi) : null; - TbQueueConsumerTask consumer = new TbQueueConsumerTask<>(key, () -> consumerCreator.apply(config, partitionId), callback); + TbQueueConsumerTask consumer = new TbQueueConsumerTask<>(key, () -> { + TbQueueConsumer queueConsumer = consumerCreator.apply(config, partitionId); + if (startOffsetProvider != null && queueConsumer instanceof TbKafkaConsumerTemplate kafkaConsumer) { + kafkaConsumer.setStartOffsetProvider(startOffsetProvider); + } + return queueConsumer; + }, callback); consumers.put(tpi, consumer); consumer.subscribe(Set.of(tpi)); launchConsumer(consumer); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java index f25a98adf4..0de1e53753 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java @@ -33,6 +33,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Function; @Slf4j public class PartitionedQueueConsumerManager extends MainQueueConsumerManager { @@ -57,7 +58,7 @@ public class PartitionedQueueConsumerManager extends MainQ protected void processTask(TbQueueConsumerManagerTask task) { if (task instanceof AddPartitionsTask addPartitionsTask) { log.info("[{}] Added partitions: {}", queueKey, addPartitionsTask.partitions()); - consumerWrapper.addPartitions(addPartitionsTask.partitions(), addPartitionsTask.onStop()); + consumerWrapper.addPartitions(addPartitionsTask.partitions(), addPartitionsTask.onStop(), addPartitionsTask.startOffsetProvider()); } else if (task instanceof RemovePartitionsTask removePartitionsTask) { log.info("[{}] Removed partitions: {}", queueKey, removePartitionsTask.partitions()); consumerWrapper.removePartitions(removePartitionsTask.partitions()); @@ -76,11 +77,11 @@ public class PartitionedQueueConsumerManager extends MainQ } public void addPartitions(Set partitions) { - addPartitions(partitions, null); + addPartitions(partitions, null, null); } - public void addPartitions(Set partitions, Consumer onStop) { - addTask(new AddPartitionsTask(partitions, onStop)); + public void addPartitions(Set partitions, Consumer onStop, Function startOffsetProvider) { + addTask(new AddPartitionsTask(partitions, onStop, startOffsetProvider)); } public void removePartitions(Set partitions) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java index e0dd9b808b..a287a391af 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; public interface TbQueueConsumerManagerTask { @@ -46,7 +47,9 @@ public interface TbQueueConsumerManagerTask { } } - record AddPartitionsTask(Set partitions, Consumer onStop) implements TbQueueConsumerManagerTask { + record AddPartitionsTask(Set partitions, + Consumer onStop, + Function startOffsetProvider) implements TbQueueConsumerManagerTask { @Override public QueueTaskType getType() { return QueueTaskType.ADD_PARTITIONS; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java index 9adc6bb996..bf02afe86c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java @@ -15,13 +15,16 @@ */ package org.thingsboard.server.queue.common.state; +import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.discovery.QueueKey; +import java.util.Map; import java.util.Set; +import java.util.function.Supplier; import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; @@ -29,14 +32,21 @@ import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTop public class KafkaQueueStateService extends QueueStateService { private final PartitionedQueueConsumerManager stateConsumer; + private final Supplier> eventsStartOffsetsProvider; - public KafkaQueueStateService(PartitionedQueueConsumerManager eventConsumer, PartitionedQueueConsumerManager stateConsumer) { + @Builder + public KafkaQueueStateService(PartitionedQueueConsumerManager eventConsumer, + PartitionedQueueConsumerManager stateConsumer, + Supplier> eventsStartOffsetsProvider) { super(eventConsumer); this.stateConsumer = stateConsumer; + this.eventsStartOffsetsProvider = eventsStartOffsetsProvider; } @Override protected void addPartitions(QueueKey queueKey, Set partitions) { + Map eventsStartOffsets = eventsStartOffsetsProvider != null ? eventsStartOffsetsProvider.get() : null; // remembering the offsets before subscribing to states + Set statePartitions = withTopic(partitions, stateConsumer.getTopic()); partitionsInProgress.addAll(statePartitions); stateConsumer.addPartitions(statePartitions, statePartition -> { @@ -51,12 +61,12 @@ public class KafkaQueueStateService TopicPartitionInfo eventPartition = statePartition.withTopic(eventConsumer.getTopic()); if (this.partitions.get(queueKey).contains(eventPartition)) { - eventConsumer.addPartitions(Set.of(eventPartition)); + eventConsumer.addPartitions(Set.of(eventPartition), null, eventsStartOffsets != null ? eventsStartOffsets::get : null); } } finally { readLock.unlock(); } - }); + }, null); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 3a76d50825..7186bf7055 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -39,6 +39,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; import org.thingsboard.server.queue.util.AfterStartUp; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -559,7 +560,15 @@ public class HashPartitionService implements PartitionService { @Override public int resolvePartitionIndex(UUID entityId, int partitions) { - int hash = hash(entityId); + return resolvePartitionIndex(hash(entityId), partitions); + } + + @Override + public int resolvePartitionIndex(String key, int partitions) { + return resolvePartitionIndex(hash(key), partitions); + } + + private int resolvePartitionIndex(int hash, int partitions) { return Math.abs(hash % partitions); } @@ -725,6 +734,12 @@ public class HashPartitionService implements PartitionService { .hash().asInt(); } + private int hash(String key) { + return hashFunction.newHasher() + .putString(key, StandardCharsets.UTF_8) + .hash().asInt(); + } + public static HashFunction forName(String name) { return switch (name) { case "murmur3_32" -> Hashing.murmur3_32(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 7abd68e25f..5dda413f17 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -79,4 +79,6 @@ public interface PartitionService { int resolvePartitionIndex(UUID entityId, int partitions); + int resolvePartitionIndex(String key, int partitions); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index 5992083d85..2a0fad0643 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -103,6 +103,9 @@ public class TopicService { } public String buildTopicName(String topic) { + if (topic == null) { + return null; + } return prefix.isBlank() ? topic : prefix + "." + topic; } @@ -113,9 +116,9 @@ public class TopicService { public String buildConsumerGroupId(String servicePrefix, TenantId tenantId, String queueName, Integer partitionId) { return this.buildTopicName( servicePrefix + queueName - + (tenantId.isSysTenantId() ? "" : ("-isolated-" + tenantId)) - + "-consumer" - + suffix(partitionId)); + + (tenantId.isSysTenantId() ? "" : ("-isolated-" + tenantId)) + + "-consumer" + + suffix(partitionId)); } String suffix(Integer partitionId) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index f7a4d2abf6..cf9f27ee39 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -315,7 +315,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi ScheduledFuture> task = delayedTasks.remove(serviceId); if (task != null) { if (task.cancel(false)) { - log.debug("[{}] Recalculate partitions ignored. Service was restarted in time [{}].", + log.info("[{}] Recalculate partitions ignored. Service was restarted in time [{}].", serviceId, serviceTypesList); } else { log.debug("[{}] Going to recalculate partitions. Service was not restarted in time [{}]!", diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java index e4e1e81815..401b451f59 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java @@ -30,6 +30,10 @@ public class EdqsConfig { @Value("#{'${queue.edqs.partitioning_strategy:tenant}'.toUpperCase()}") private EdqsPartitioningStrategy partitioningStrategy; + @Value("${queue.edqs.events_topic:edqs.events}") + private String eventsTopic; + @Value("${queue.edqs.state_topic:edqs.state}") + private String stateTopic; @Value("${queue.edqs.requests_topic:edqs.requests}") private String requestsTopic; @Value("${queue.edqs.responses_topic:edqs.responses}") diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java deleted file mode 100644 index d859b50994..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.edqs; - -import lombok.Getter; - -@Getter -public enum EdqsQueue { - - EVENTS("edqs.events", false, false), - STATE("edqs.state", true, true); - - private final String topic; - private final boolean readFromBeginning; - private final boolean stopWhenRead; - - EdqsQueue(String topic, boolean readFromBeginning, boolean stopWhenRead) { - this.topic = topic; - this.readFromBeginning = readFromBeginning; - this.stopWhenRead = stopWhenRead; - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java index b5541c740b..5c0d68779a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java @@ -25,11 +25,13 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; public interface EdqsQueueFactory { - TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue); + TbQueueConsumer> createEdqsEventsConsumer(); - TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group); + TbQueueConsumer> createEdqsEventsToBackupConsumer(); - TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue); + TbQueueConsumer> createEdqsStateConsumer(); + + TbQueueProducer> createEdqsStateProducer(); TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java index 0801399c14..8c670e66c0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java @@ -43,24 +43,23 @@ public class InMemoryEdqsQueueFactory implements EdqsQueueFactory { private final TbQueueAdmin queueAdmin; @Override - public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue) { - if (queue == EdqsQueue.STATE) { - throw new UnsupportedOperationException(); - } - return new InMemoryTbQueueConsumer<>(storage, queue.getTopic()); + public TbQueueConsumer> createEdqsEventsConsumer() { + return new InMemoryTbQueueConsumer<>(storage, edqsConfig.getEventsTopic()); } @Override - public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group) { - return createEdqsMsgConsumer(queue); + public TbQueueConsumer> createEdqsEventsToBackupConsumer() { + throw new UnsupportedOperationException(); } @Override - public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { - if (queue == EdqsQueue.STATE) { - throw new UnsupportedOperationException(); - } - return new InMemoryTbQueueProducer<>(storage, queue.getTopic()); + public TbQueueConsumer> createEdqsStateConsumer() { + throw new UnsupportedOperationException(); + } + + @Override + public TbQueueProducer> createEdqsStateProducer() { + throw new UnsupportedOperationException(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java index e985696040..ab88943b10 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java @@ -19,11 +19,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueResponseTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; @@ -71,54 +68,68 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { } @Override - public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue) { - String consumerGroup = "edqs-" + queue.name().toLowerCase() + "-consumer-group-" + serviceInfoProvider.getServiceId(); - return createEdqsMsgConsumer(queue, consumerGroup); + public TbKafkaConsumerTemplate> createEdqsEventsConsumer() { + return createEdqsMsgConsumer(edqsConfig.getEventsTopic(), + "edqs-events-" + consumerCounter.getAndIncrement() + "-consumer-" + serviceInfoProvider.getServiceId(), + null, // not using consumer group management, offsets from the edqs-events-to-backup-consumer-group are used (see KafkaEdqsStateService) + false, edqsEventsAdmin); } @Override - public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group) { + public TbKafkaConsumerTemplate> createEdqsEventsToBackupConsumer() { + return createEdqsMsgConsumer(edqsConfig.getEventsTopic(), + "edqs-events-to-backup-consumer-" + serviceInfoProvider.getServiceId(), + "edqs-events-to-backup-consumer-group", + false, edqsEventsAdmin); + } + + @Override + public TbKafkaConsumerTemplate> createEdqsStateConsumer() { + return createEdqsMsgConsumer(edqsConfig.getStateTopic(), + "edqs-state-" + consumerCounter.getAndIncrement() + "-consumer-" + serviceInfoProvider.getServiceId(), + null, // not using consumer group management + true, edqsStateAdmin); + } + + public TbKafkaConsumerTemplate> createEdqsMsgConsumer(String topic, String clientId, String group, boolean readFullAndStop, TbKafkaAdmin admin) { return TbKafkaConsumerTemplate.>builder() .settings(kafkaSettings) - .topic(topicService.buildTopicName(queue.getTopic())) - .readFromBeginning(queue.isReadFromBeginning()) - .stopWhenRead(queue.isStopWhenRead()) - .clientId("edqs-" + queue.name().toLowerCase() + "-" + consumerCounter.getAndIncrement() + "-consumer-" + serviceInfoProvider.getServiceId()) + .topic(topicService.buildTopicName(topic)) + .readFromBeginning(readFullAndStop) + .stopWhenRead(readFullAndStop) + .clientId(clientId) .groupId(topicService.buildTopicName(group)) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) - .admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin) + .admin(admin) .statsService(consumerStatsService) .build(); } @Override - public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + public TbQueueProducer> createEdqsStateProducer() { return TbKafkaProducerTemplate.>builder() - .clientId("edqs-" + queue.name().toLowerCase() + "-producer-" + serviceInfoProvider.getServiceId()) + .clientId("edqs-state-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getStateTopic())) .settings(kafkaSettings) - .admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin) + .admin(edqsStateAdmin) .build(); } @Override public TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate() { - String requestsConsumerGroup = "edqs-requests-consumer-group-" + edqsConfig.getLabel(); - var requestConsumer = TbKafkaConsumerTemplate.>builder() - .settings(kafkaSettings) - .topic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) - .clientId("edqs-requests-consumer-" + serviceInfoProvider.getServiceId()) - .groupId(topicService.buildTopicName(requestsConsumerGroup)) - .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) - .admin(edqsRequestsAdmin) - .statsService(consumerStatsService); + var requestConsumer = createEdqsMsgConsumer(edqsConfig.getRequestsTopic(), + "edqs-requests-consumer-" + serviceInfoProvider.getServiceId(), + "edqs-requests-consumer-group", + false, edqsRequestsAdmin); var responseProducer = TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) .clientId("edqs-response-producer-" + serviceInfoProvider.getServiceId()) .defaultTopic(topicService.buildTopicName(edqsConfig.getResponsesTopic())) - .admin(edqsRequestsAdmin); + .admin(edqsRequestsAdmin) + .build(); return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() - .requestTemplate(requestConsumer.build()) - .responseTemplate(responseProducer.build()) + .requestTemplate(requestConsumer) + .responseTemplate(responseProducer) .maxPendingRequests(edqsConfig.getMaxPendingRequests()) .requestTimeout(edqsConfig.getMaxRequestTimeout()) .pollInterval(edqsConfig.getPollInterval()) @@ -128,7 +139,7 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { } @Override - public TbQueueAdmin getEdqsQueueAdmin() { + public TbKafkaAdmin getEdqsQueueAdmin() { return edqsEventsAdmin; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index 4ae744be67..37f881b49c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.queue.kafka; +import lombok.Getter; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.CreateTopicsResult; import org.apache.kafka.clients.admin.ListOffsetsResult; @@ -45,6 +47,7 @@ public class TbKafkaAdmin implements TbQueueAdmin { private final TbKafkaSettings settings; private final Map topicConfigs; + @Getter private final int numPartitions; private volatile Set topics; @@ -157,8 +160,7 @@ public class TbKafkaAdmin implements TbQueueAdmin { if (partitionId == null) { return; } - Map oldOffsets = - settings.getAdminClient().listConsumerGroupOffsets(fatGroupId).partitionsToOffsetAndMetadata().get(10, TimeUnit.SECONDS); + Map oldOffsets = getConsumerGroupOffsets(fatGroupId); if (oldOffsets.isEmpty()) { return; } @@ -169,8 +171,7 @@ public class TbKafkaAdmin implements TbQueueAdmin { continue; } var om = consumerOffset.getValue(); - Map newOffsets = - settings.getAdminClient().listConsumerGroupOffsets(newGroupId).partitionsToOffsetAndMetadata().get(10, TimeUnit.SECONDS); + Map newOffsets = getConsumerGroupOffsets(newGroupId); var existingOffset = newOffsets.get(tp); if (existingOffset == null) { @@ -187,32 +188,52 @@ public class TbKafkaAdmin implements TbQueueAdmin { } } + @SneakyThrows + public Map getConsumerGroupOffsets(String groupId) { + return settings.getAdminClient().listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata().get(10, TimeUnit.SECONDS); + } + public boolean isTopicEmpty(String topic) { + return areAllTopicsEmpty(Set.of(topic)); + } + + public boolean areAllTopicsEmpty(Set topics) { try { - if (!getTopics().contains(topic)) { + List existingTopics = getTopics().stream().filter(topics::contains).toList(); + if (existingTopics.isEmpty()) { return true; } - TopicDescription topicDescription = settings.getAdminClient().describeTopics(Collections.singletonList(topic)).topicNameValues().get(topic).get(); - List partitions = topicDescription.partitions().stream().map(partitionInfo -> new TopicPartition(topic, partitionInfo.partition())).toList(); - Map beginningOffsets = settings.getAdminClient().listOffsets(partitions.stream() + List allPartitions = settings.getAdminClient().describeTopics(existingTopics).topicNameValues().entrySet().stream() + .flatMap(entry -> { + String topic = entry.getKey(); + TopicDescription topicDescription; + try { + topicDescription = entry.getValue().get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + return topicDescription.partitions().stream().map(partitionInfo -> new TopicPartition(topic, partitionInfo.partition())); + }) + .toList(); + + Map beginningOffsets = settings.getAdminClient().listOffsets(allPartitions.stream() .collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.earliest()))).all().get(); - - Map endOffsets = settings.getAdminClient().listOffsets(partitions.stream() + Map endOffsets = settings.getAdminClient().listOffsets(allPartitions.stream() .collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.latest()))).all().get(); - for (TopicPartition partition : partitions) { + for (TopicPartition partition : allPartitions) { long beginningOffset = beginningOffsets.get(partition).offset(); long endOffset = endOffsets.get(partition).offset(); if (beginningOffset != endOffset) { - log.debug("Partition [{}] of topic [{}] is not empty. Returning false.", partition.partition(), topic); + log.debug("Partition [{}] of topic [{}] is not empty. Returning false.", partition.partition(), partition.topic()); return false; } } return true; } catch (InterruptedException | ExecutionException e) { - log.error("Failed to check if topic [{}] is empty.", topic, e); + log.error("Failed to check if topics [{}] empty.", topics, e); return false; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 4bd3bf0fe6..3abed8475a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -16,6 +16,8 @@ package org.thingsboard.server.queue.kafka; import lombok.Builder; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; @@ -39,7 +41,9 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; /** * Created by ashvayka on 24.09.18. @@ -47,13 +51,16 @@ import java.util.stream.Collectors; @Slf4j public class TbKafkaConsumerTemplate extends AbstractTbQueueConsumerTemplate, T> { - private final TbQueueAdmin admin; + private final TbKafkaAdmin admin; private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; private final TbKafkaConsumerStatsService statsService; + @Getter private final String groupId; + @Setter + private Function startOffsetProvider; private final boolean readFromBeginning; // reset offset to beginning private final boolean stopWhenRead; // stop consuming when reached end offset remembered on start private int readCount; @@ -78,7 +85,7 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue statsService.registerClientGroup(groupId); } - this.admin = admin; + this.admin = (TbKafkaAdmin) admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; this.readFromBeginning = readFromBeginning; @@ -105,14 +112,19 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue List toSubscribe = new ArrayList<>(); topics.forEach((topic, kafkaPartitions) -> { if (kafkaPartitions == null) { - toSubscribe.add(topic); - } else { - List topicPartitions = kafkaPartitions.stream() - .map(partition -> new TopicPartition(topic, partition)) - .toList(); - consumer.assign(topicPartitions); - onPartitionsAssigned(topicPartitions); + if (groupId != null) { + toSubscribe.add(topic); + return; + } else { // if no consumer group management - manually assigning all topic partitions + kafkaPartitions = IntStream.range(0, admin.getNumPartitions()).boxed().toList(); + } } + + List topicPartitions = kafkaPartitions.stream() + .map(partition -> new TopicPartition(topic, partition)) + .toList(); + consumer.assign(topicPartitions); + onPartitionsAssigned(topicPartitions); }); if (!toSubscribe.isEmpty()) { if (readFromBeginning || stopWhenRead) { @@ -179,9 +191,21 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private void onPartitionsAssigned(Collection partitions) { if (readFromBeginning) { + log.debug("Seeking to beginning for {}", partitions); consumer.seekToBeginning(partitions); + } else if (startOffsetProvider != null) { + partitions.forEach(topicPartition -> { + Long offset = startOffsetProvider.apply(topicPartition.topic()); + if (offset != null) { + log.debug("Seeking to offset {} for {}", offset, topicPartition); + consumer.seek(topicPartition, offset); + } else { + log.info("No start offset provided for {}", topicPartition); + } + }); } if (stopWhenRead) { + log.debug("Getting end offsets for {}", partitions); endOffsets = consumer.endOffsets(partitions).entrySet().stream() .filter(entry -> entry.getValue() > 0) .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); @@ -195,7 +219,9 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue @Override protected void doCommit() { - consumer.commitSync(); + if (groupId != null) { + consumer.commitSync(); + } } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java index 95be49f82b..f2309f7224 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java @@ -20,11 +20,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.edqs.EdqsQueue; public interface EdqsClientQueueFactory { - TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue); + TbQueueProducer> createEdqsEventsProducer(); TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index fac2cf5d5e..89d83af826 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -37,7 +37,6 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsConfig; -import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @@ -149,7 +148,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + public TbQueueConsumer> createToCalculatedFieldNotificationMsgConsumer() { return new InMemoryTbQueueConsumer<>(storage, topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } @@ -239,8 +238,8 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { - return new InMemoryTbQueueProducer<>(storage, queue.getTopic()); + public TbQueueProducer> createEdqsEventsProducer() { + return new InMemoryTbQueueProducer<>(storage, edqsConfig.getEventsTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index f07bd9dcbb..dcebe085b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -54,7 +54,6 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsConfig; -import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -542,7 +541,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + public TbQueueConsumer> createToCalculatedFieldNotificationMsgConsumer() { TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); @@ -590,9 +589,10 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + public TbQueueProducer> createEdqsEventsProducer() { return TbKafkaProducerTemplate.>builder() - .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .clientId("edqs-events-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getEventsTopic())) .settings(kafkaSettings) .admin(edqsEventsAdmin) .build(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 3c6d144a0c..e9c42b0022 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -52,7 +52,6 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsConfig; -import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -480,9 +479,10 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + public TbQueueProducer> createEdqsEventsProducer() { return TbKafkaProducerTemplate.>builder() - .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .clientId("edqs-events-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getEventsTopic())) .settings(kafkaSettings) .admin(edqsEventsAdmin) .build(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index d0e6c2f123..d43ef5c9ac 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -48,7 +48,7 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -79,6 +79,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; private final TbQueueCalculatedFieldSettings calculatedFieldSettings; + private final EdqsConfig edqsConfig; private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; @@ -101,7 +102,9 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, TbQueueCalculatedFieldSettings calculatedFieldSettings, + TbQueueEdgeSettings edgeSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, + EdqsConfig edqsConfig, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -113,6 +116,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; this.calculatedFieldSettings = calculatedFieldSettings; + this.edqsConfig = edqsConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -337,7 +341,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + public TbQueueConsumer> createToCalculatedFieldNotificationMsgConsumer() { TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); @@ -349,6 +353,16 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { return consumerBuilder.build(); } + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-calculated-field-notifications-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + @Override public TbQueueConsumer> createCalculatedFieldStateConsumer() { return TbKafkaConsumerTemplate.>builder() @@ -375,9 +389,10 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + public TbQueueProducer> createEdqsEventsProducer() { return TbKafkaProducerTemplate.>builder() - .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .clientId("edqs-events-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getEventsTopic())) .settings(kafkaSettings) .admin(edqsEventsAdmin) .build(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index dcadf02d02..8e1952fc14 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -50,6 +50,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toEdgeNotifications; private TbQueueProducer> toEdgeEvents; private TbQueueProducer> toCalculatedFields; + private TbQueueProducer> toCalculatedFieldNotifications; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -68,6 +69,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); this.toCalculatedFields = tbQueueProvider.createToCalculatedFieldMsgProducer(); + this.toCalculatedFieldNotifications = tbQueueProvider.createToCalculatedFieldNotificationMsgProducer(); } @Override @@ -132,7 +134,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { @Override public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Rule Engine Service!"); + return toCalculatedFieldNotifications; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 03f662d8bb..329e3e346a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -127,7 +127,9 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueProducer> createToCalculatedFieldMsgProducer(); - TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer(); + TbQueueConsumer> createToCalculatedFieldNotificationMsgConsumer(); + + TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); TbQueueConsumer> createCalculatedFieldStateConsumer(); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 95fff48883..72792c1093 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -15,6 +15,7 @@ */ package org.thingsboard.script.api.tbel; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.primitives.Bytes; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ArrayUtils; @@ -23,6 +24,10 @@ import org.mvel2.ParserConfiguration; import org.mvel2.execution.ExecutionArrayList; import org.mvel2.execution.ExecutionHashMap; import org.mvel2.util.MethodStub; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.geo.Coordinates; +import org.thingsboard.common.util.geo.GeoUtil; +import org.thingsboard.common.util.geo.RangeUnit; import org.thingsboard.server.common.data.StringUtils; import java.io.IOException; @@ -371,6 +376,10 @@ public class TbUtils { byte[].class, int.class))); parserConfig.addImport("parseBinaryArrayToInt", new MethodStub(TbUtils.class.getMethod("parseBinaryArrayToInt", byte[].class, int.class, int.class))); + parserConfig.addImport("isInsidePolygon", new MethodStub(TbUtils.class.getMethod("isInsidePolygon", + double.class, double.class, String.class))); + parserConfig.addImport("isInsideCircle", new MethodStub(TbUtils.class.getMethod("isInsideCircle", + double.class, double.class, String.class))); } public static String btoa(String input) { @@ -1437,6 +1446,22 @@ public class TbUtils { return result; } + public static boolean isInsidePolygon(double latitude, double longitude, String perimeter) { + return GeoUtil.contains(perimeter, new Coordinates(latitude, longitude)); + } + + public static boolean isInsideCircle(double latitude, double longitude, String perimeter) { + JsonNode perimeterJson = JacksonUtil.toJsonNode(perimeter); + double centerLatitude = Double.parseDouble(perimeterJson.get("latitude").asText()); + double centerLongitude = Double.parseDouble(perimeterJson.get("longitude").asText()); + double range = Double.parseDouble(perimeterJson.get("radius").asText()); + RangeUnit rangeUnit = perimeterJson.has("radiusUnit") ? RangeUnit.valueOf(perimeterJson.get("radiusUnit").asText()) : RangeUnit.METER; + + Coordinates entityCoordinates = new Coordinates(latitude, longitude); + Coordinates perimeterCoordinates = new Coordinates(centerLatitude, centerLongitude); + return range > GeoUtil.distance(entityCoordinates, perimeterCoordinates, rangeUnit); + } + private static byte isValidIntegerToByte(Integer val) { if (val > 255 || val < -128) { throw new NumberFormatException("The value '" + val + "' could not be correctly converted to a byte. " + diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index e3860ed89a..1063a8e8de 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -1150,6 +1150,26 @@ public class TbUtilsTest { Assertions.assertTrue(TbUtils.isNaN(Double.NaN)); } + @Test + public void isInsidePolygon() { + // outside the polygon + String perimeter = "[[[50.75581142688204,29.097910166341073],[50.16785158177623,29.35066098977171],[50.164329922384674,29.773743889862114],[50.16785158177623,30.801230932938843],[50.459245308833495,30.92760634465418],[50.486522489629564,30.68548421850448],[50.703612031034005,30.872660513473573]],[[50.606017492632766,29.36165015600782],[50.54317104075835,29.762754723626013],[50.41021974600505,29.455058069014804]]]"; + Assertions.assertFalse(TbUtils.isInsidePolygon(50.50869555168039, 30.80123093293884, perimeter)); + // inside the polygon + Assertions.assertTrue(TbUtils.isInsidePolygon(50.50520628167696, 30.339685951022016, perimeter)); + // inside the hole + Assertions.assertFalse(TbUtils.isInsidePolygon(50.52265651287081, 29.488025567723156, perimeter)); + } + + @Test + public void isInsideCircle() { + // outside the circle + String perimeter = "{\"latitude\":50.32254778825905,\"longitude\":28.207787701215757,\"radius\":47477.33130420423}"; + Assertions.assertFalse(TbUtils.isInsideCircle(50.81490715736681, 28.05943395702824, perimeter)); + // inside the circle + Assertions.assertTrue(TbUtils.isInsideCircle(50.599397971892444, 28.086906872618542, perimeter)); + } + private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index a7c4cb8c6c..5aea93528a 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -762,7 +762,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement ctx.writeAndFlush(createSubAckMessage(mqttMsg.variableHeader().messageId(), Collections.singletonList(MqttReasonCodes.SubAck.NOT_AUTHORIZED.byteValue() & 0xFF))); return; } - log.trace("[{}] Processing subscription [{}]!", sessionId, mqttMsg.variableHeader().messageId()); + //TODO consume the rate limit + log.trace("[{}][{}] Processing subscription [{}]!", deviceSessionCtx.getTenantId(), sessionId, mqttMsg.variableHeader().messageId()); List grantedQoSList = new ArrayList<>(); boolean activityReported = false; for (MqttTopicSubscription subscription : mqttMsg.payload().topicSubscriptions()) { @@ -772,7 +773,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement if (MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC.equals(topic)) { registerSubQoS(topic, grantedQoSList, reqQoS); } else { - log.debug("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS); + log.debug("[{}][{}] Failed to subscribe because this session is provision only [{}][{}]", deviceSessionCtx.getTenantId(), sessionId, topic, reqQoS); grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID)); } activityReported = true; @@ -847,13 +848,14 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement registerSubQoS(topic, grantedQoSList, reqQoS); break; default: - log.warn("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS); + //TODO increment an error counter if any exists + log.warn("[{}][{}] Failed to subscribe because topic is not supported [{}][{}]", deviceSessionCtx.getTenantId(), sessionId, topic, reqQoS); grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID)); break; } } } catch (Exception e) { - log.warn("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS, e); + log.warn("[{}][{}] Failed to subscribe to [{}][{}]", deviceSessionCtx.getTenantId(), sessionId, topic, reqQoS, e); grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.IMPLEMENTATION_SPECIFIC_ERROR)); } } diff --git a/common/util/pom.xml b/common/util/pom.xml index 430a6e7df9..82768cdb68 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -103,6 +103,14 @@ com.fasterxml.jackson.datatype jackson-datatype-jdk8 + + org.locationtech.spatial4j + spatial4j + + + org.locationtech.jts + jts-core + diff --git a/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java b/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java index c4b062a0d8..1b035f61f1 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java @@ -22,7 +22,7 @@ import java.util.Set; public final class DebugModeUtil { - private static final int DEBUG_MODE_DEFAULT_DURATION_MINUTES = 15; + public static final int DEBUG_MODE_DEFAULT_DURATION_MINUTES = 15; private DebugModeUtil() { } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/Coordinates.java b/common/util/src/main/java/org/thingsboard/common/util/geo/Coordinates.java similarity index 94% rename from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/Coordinates.java rename to common/util/src/main/java/org/thingsboard/common/util/geo/Coordinates.java index c3b91e39bb..dc1998bff1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/Coordinates.java +++ b/common/util/src/main/java/org/thingsboard/common/util/geo/Coordinates.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.rule.engine.geo; +package org.thingsboard.common.util.geo; import lombok.Data; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/GeoUtil.java b/common/util/src/main/java/org/thingsboard/common/util/geo/GeoUtil.java similarity index 99% rename from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/GeoUtil.java rename to common/util/src/main/java/org/thingsboard/common/util/geo/GeoUtil.java index f92a345362..11a6799348 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/GeoUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/geo/GeoUtil.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.rule.engine.geo; +package org.thingsboard.common.util.geo; import com.google.gson.JsonArray; import com.google.gson.JsonElement; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/Perimeter.java b/common/util/src/main/java/org/thingsboard/common/util/geo/Perimeter.java similarity index 95% rename from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/Perimeter.java rename to common/util/src/main/java/org/thingsboard/common/util/geo/Perimeter.java index 3474683c6c..238fa19af5 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/Perimeter.java +++ b/common/util/src/main/java/org/thingsboard/common/util/geo/Perimeter.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.rule.engine.geo; +package org.thingsboard.common.util.geo; import lombok.Data; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/PerimeterType.java b/common/util/src/main/java/org/thingsboard/common/util/geo/PerimeterType.java similarity index 94% rename from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/PerimeterType.java rename to common/util/src/main/java/org/thingsboard/common/util/geo/PerimeterType.java index 2df6cf796c..c3b334abd7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/PerimeterType.java +++ b/common/util/src/main/java/org/thingsboard/common/util/geo/PerimeterType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.rule.engine.geo; +package org.thingsboard.common.util.geo; public enum PerimeterType { CIRCLE, POLYGON diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/RangeUnit.java b/common/util/src/main/java/org/thingsboard/common/util/geo/RangeUnit.java similarity index 95% rename from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/RangeUnit.java rename to common/util/src/main/java/org/thingsboard/common/util/geo/RangeUnit.java index 090d251b33..56883ca43f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/RangeUnit.java +++ b/common/util/src/main/java/org/thingsboard/common/util/geo/RangeUnit.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.rule.engine.geo; +package org.thingsboard.common.util.geo; public enum RangeUnit { METER(1000.0), KILOMETER(1.0), FOOT(3280.84), MILE(0.62137), NAUTICAL_MILE(0.539957); diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 187de20667..c9c7af1a89 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -39,6 +39,7 @@ public class CalculatedFieldDataValidator extends DataValidator protected void validateCreate(TenantId tenantId, CalculatedField calculatedField) { validateNumberOfCFsPerEntity(tenantId, calculatedField.getEntityId()); validateNumberOfArgumentsPerCF(tenantId, calculatedField); + validateArgumentNames(calculatedField); } @Override @@ -48,6 +49,7 @@ public class CalculatedFieldDataValidator extends DataValidator throw new DataValidationException("Can't update non existing calculated field!"); } validateNumberOfArgumentsPerCF(tenantId, calculatedField); + validateArgumentNames(calculatedField); return old; } @@ -71,4 +73,10 @@ public class CalculatedFieldDataValidator extends DataValidator } } + private void validateArgumentNames(CalculatedField calculatedField) { + if (calculatedField.getConfiguration().getArguments().containsKey("ctx")) { + throw new DataValidationException("Argument name 'ctx' is reserved and cannot be used."); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 4f7239cfb3..1c5d554350 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -119,7 +119,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq @Override public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { var aggParams = query.getAggParameters(); - if (Aggregation.NONE.equals(aggParams.getAggregation())) { + if (Aggregation.NONE.equals(aggParams.getAggregation()) || aggParams.getInterval() < 1) { return Futures.immediateFuture(findAllAsyncWithLimit(entityId, query)); } else { List>> futures = new ArrayList<>(); @@ -144,7 +144,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } } - private ReadTsKvQueryResult findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + ReadTsKvQueryResult findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { Integer keyId = keyDictionaryDao.getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index 04a57c3fea..e165d6cfe6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -44,6 +44,7 @@ import java.util.List; import java.util.Optional; import java.util.UUID; +import static org.thingsboard.common.util.DebugModeUtil.DEBUG_MODE_DEFAULT_DURATION_MINUTES; import static org.thingsboard.server.dao.service.Validator.validateId; @Service("TenantProfileDaoService") @@ -160,7 +161,9 @@ public class TenantProfileServiceImpl extends AbstractCachedEntityService maxTsIntervals || intervalCounts < 0) { throw new IncorrectParameterException("Incorrect TsKvQuery. Number of intervals is to high - " + intervalCounts + ". " + diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index a070de4d82..6a38159804 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.objects.TelemetryEntityView; import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.AbstractServiceTest; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -757,6 +758,21 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { assertThat(fullList).containsOnlyOnceElementsOf(timeseries); } + @Test + public void testFindAllByQueriesWithAggregationAndZeroInterval() throws Exception { + testFindAllByQueriesWithAggregationAndInvalidInterval(0); + } + + @Test + public void testFindAllByQueriesWithAggregationAndNegativeInterval() throws Exception { + testFindAllByQueriesWithAggregationAndInvalidInterval(-1); + } + + private void testFindAllByQueriesWithAggregationAndInvalidInterval(long interval) { + BaseReadTsKvQuery query = new BaseReadTsKvQuery(STRING_KEY, TS, TS, interval, 1000, Aggregation.SUM, "DESC"); + Assert.assertThrows(IncorrectParameterException.class, () -> findAndVerifyQueryId(deviceId, query)); + } + private TsKvEntry save(DeviceId deviceId, long ts, long value) throws Exception { TsKvEntry entry = new BasicTsKvEntry(ts, new LongDataEntry(LONG_KEY, value)); tsService.save(tenantId, deviceId, entry).get(MAX_TIMEOUT, TimeUnit.SECONDS); diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index f9ea7d2a41..eb34e6dc4c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -51,6 +51,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { Optional optionalListenableFuture = Optional.of(mock(TsKvEntry.class)); willReturn(Futures.immediateFuture(optionalListenableFuture)).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); willReturn(Futures.immediateFuture(mock(ReadTsKvQueryResult.class))).given(tsDao).getReadTsKvQueryResultFuture(any(), any()); + willReturn(mock(ReadTsKvQueryResult.class)).given(tsDao).findAllAsyncWithLimit(any(), any()); } @Test @@ -146,6 +147,24 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } } + @Test + public void givenZeroInterval_whenAggregateCount_thenFindAllWithoutAggregation() { + givenInterval_whenAggregateCount_thenFindAllWithoutAggregation(0); + } + + @Test + public void givenNegativeInterval_whenAggregateCount_thenFindAllWithoutAggregation() { + givenInterval_whenAggregateCount_thenFindAllWithoutAggregation(-1); + } + + public void givenInterval_whenAggregateCount_thenFindAllWithoutAggregation(int interval) { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, interval, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAllAsyncWithLimit(any(), any()); + verify(tsDao, times(0)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + } + long getTsForReadTsKvQuery(long startTs, long endTs) { return startTs + (endTs - startTs) / 2L; } diff --git a/docker/docker-compose.edqs.volumes.yml b/docker/docker-compose.edqs.volumes.yml index 89b4b3a59c..3a45542b99 100644 --- a/docker/docker-compose.edqs.volumes.yml +++ b/docker/docker-compose.edqs.volumes.yml @@ -17,10 +17,10 @@ version: '3.0' services: - tb-edqs-1: + tb-edqs1: volumes: - tb-edqs-log-volume:/var/log/tb-edqs - tb-edqs-2: + tb-edqs2: volumes: - tb-edqs-log-volume:/var/log/tb-edqs diff --git a/docker/docker-compose.edqs.yml b/docker/docker-compose.edqs.yml index 6dd9606ee6..21bace143a 100644 --- a/docker/docker-compose.edqs.yml +++ b/docker/docker-compose.edqs.yml @@ -29,9 +29,12 @@ services: tb-rule-engine2: env_file: - tb-rule-engine-edqs.env - tb-edqs-1: + tb-edqs1: restart: always image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + environment: + TB_SERVICE_ID: tb-edqs1 + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-edqs.env volumes: @@ -42,9 +45,12 @@ services: depends_on: - zookeeper - kafka - tb-edqs-2: + tb-edqs2: restart: always image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + environment: + TB_SERVICE_ID: tb-edqs2 + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-edqs.env volumes: diff --git a/docker/tb-edqs/conf/logback.xml b/docker/tb-edqs/conf/logback.xml index 14b5b0f04b..c8e97e48e4 100644 --- a/docker/tb-edqs/conf/logback.xml +++ b/docker/tb-edqs/conf/logback.xml @@ -24,7 +24,7 @@ /var/log/tb-edqs/${TB_SERVICE_ID}/tb-edqs.log - /var/log/tb-edqs/tb-edqs.%d{yyyy-MM-dd}.%i.log + /var/log/tb-edqs/${TB_SERVICE_ID}/tb-edqs.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB @@ -41,7 +41,6 @@ - diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index f7d0eda841..353d76391b 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -57,6 +57,10 @@ queue: partitions: "${TB_EDQS_PARTITIONS:12}" # EDQS partitioning strategy: tenant (partitions are resolved and distributed by tenant id) or none (partitions are resolved by message key; each instance has all the partitions) partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" + # EDQS events topic + events_topic: "${TB_EDQS_EVENTS_TOPIC:edqs.events}" + # EDQS state topic + state_topic: "${TB_EDQS_STATE_TOPIC:edqs.state}" # EDQS requests topic requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" # EDQS responses topic @@ -148,12 +152,12 @@ queue: # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) topic-properties: - # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions - edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" - # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions - edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" - # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions - edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" + # Kafka properties for EDQS events topics + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:86400000;segment.bytes:52428800;retention.bytes:-1;partitions:1;min.insync.replicas:1}" + # Kafka properties for EDQS requests topic (default: 3 minutes retention) + edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for EDQS state topic (infinite retention, compaction) + edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:1;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/Latencies.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/Latencies.java index a190b7b791..b42e9138e4 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/Latencies.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/Latencies.java @@ -20,6 +20,7 @@ public class Latencies { public static final String WS_CONNECT = "wsConnect"; public static final String WS_SUBSCRIBE = "wsSubscribe"; public static final String LOG_IN = "logIn"; + public static final String EDQS_QUERY = "edqsQuery"; public static String request(String key) { return String.format("%sRequest", key); diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java index 342ee121ef..9c3ee5b786 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java @@ -18,5 +18,6 @@ package org.thingsboard.monitoring.data; public class MonitoredServiceKey { public static final String GENERAL = "Monitoring"; + public static final String EDQS = "*EDQS*"; } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java index 9157e9f31c..7219e4f9aa 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java @@ -15,10 +15,13 @@ */ package org.thingsboard.monitoring.service; +import com.google.common.collect.Sets; import jakarta.annotation.PostConstruct; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationContext; import org.thingsboard.monitoring.client.TbClient; import org.thingsboard.monitoring.client.WsClient; @@ -27,13 +30,26 @@ import org.thingsboard.monitoring.config.MonitoringConfig; import org.thingsboard.monitoring.config.MonitoringTarget; import org.thingsboard.monitoring.data.Latencies; import org.thingsboard.monitoring.data.MonitoredServiceKey; +import org.thingsboard.monitoring.data.ServiceFailureException; import org.thingsboard.monitoring.service.transport.TransportHealthChecker; import org.thingsboard.monitoring.util.TbStopWatch; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.TsValue; import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -41,6 +57,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j public abstract class BaseMonitoringService, T extends MonitoringTarget> { @@ -61,6 +78,9 @@ public abstract class BaseMonitoringService, T ext @Autowired protected ApplicationContext applicationContext; + @Value("${monitoring.edqs.enabled:false}") + private boolean edqsMonitoringEnabled; + @PostConstruct private void init() { if (configs == null || configs.isEmpty()) { @@ -108,6 +128,21 @@ public abstract class BaseMonitoringService, T ext check(healthChecker, wsClient); } } + + if (edqsMonitoringEnabled) { + try { + stopWatch.start(); + checkEdqs(); + reporter.reportLatency(Latencies.EDQS_QUERY, stopWatch.getTime()); + + reporter.serviceIsOk(MonitoredServiceKey.EDQS); + } catch (ServiceFailureException e) { + reporter.serviceFailure(MonitoredServiceKey.EDQS, e); + } catch (Exception e) { + reporter.serviceFailure(MonitoredServiceKey.GENERAL, e); + } + } + reporter.reportLatencies(tbClient); log.debug("Finished {}", getName()); } catch (Throwable error) { @@ -149,6 +184,39 @@ public abstract class BaseMonitoringService, T ext } } + private void checkEdqs() { + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(EntityType.DEVICE); + EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"))); + EntityDataQuery entityDataQuery = new EntityDataQuery(entityTypeFilter, pageLink, + List.of(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "type")), + List.of(new EntityKey(EntityKeyType.TIME_SERIES, "testData")), + Collections.emptyList()); + + PageData result = tbClient.findEntityDataByQuery(entityDataQuery); + Set devices = result.getData().stream() + .map(entityData -> entityData.getEntityId().getId()) + .collect(Collectors.toSet()); + Set missing = Sets.difference(new HashSet<>(this.devices), devices); + if (!missing.isEmpty()) { + throw new ServiceFailureException("Missing devices in the response: " + missing); + } + + result.getData().stream() + .filter(entityData -> this.devices.contains(entityData.getEntityId().getId())) + .forEach(entityData -> { + Map values = new HashMap<>(entityData.getLatest().get(EntityKeyType.ENTITY_FIELD)); + values.putAll(entityData.getLatest().get(EntityKeyType.TIME_SERIES)); + + Stream.of("name", "type", "testData").forEach(key -> { + TsValue value = values.get(key); + if (value == null || StringUtils.isBlank(value.getValue())) { + throw new ServiceFailureException("Missing " + key + " for device " + entityData.getEntityId()); + } + }); + }); + } + @SneakyThrows private Set getAssociatedUrls(String baseUrl) { URI url = new URI(baseUrl); diff --git a/monitoring/src/main/resources/tb-monitoring.yml b/monitoring/src/main/resources/tb-monitoring.yml index ae0d265ed6..6cc2e79cb4 100644 --- a/monitoring/src/main/resources/tb-monitoring.yml +++ b/monitoring/src/main/resources/tb-monitoring.yml @@ -105,6 +105,9 @@ monitoring: # To add more targets, use following environment variables: # monitoring.transports.lwm2m.targets[1].base_url, monitoring.transports.lwm2m.targets[2].base_url, etc. + edqs: + enabled: "${EDQS_MONITORING_ENABLED:false}" + notifications: message_prefix: '${NOTIFICATION_MESSAGE_PREFIX:}' slack: diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 65def6a964..f2e721bdcb 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -188,8 +188,8 @@ public class ContainerTestSuite { .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-edqs-1", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-edqs-2", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); + .waitingFor("tb-edqs1", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-edqs2", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); testContainer.start(); setActive(true); } catch (Exception e) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java index 63faed41ab..2593e18a55 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; @@ -73,7 +74,7 @@ public class CalculatedFieldTest extends AbstractContainerTest { " var airDensity = pressure / (287.05 * temperatureK);\n" + "\n" + " return {\n" + - " \"airDensity\": airDensity\n" + + " \"airDensity\": toFixed(airDensity, 2)\n" + " };"; private TenantId tenantId; @@ -280,6 +281,34 @@ public class CalculatedFieldTest extends AbstractContainerTest { testRestClient.deleteCalculatedFieldIfExists(savedCalculatedField.getId()); } + @Test + public void testEntityIdIsProfileAndRefEntityIsCommon() { + // login tenant admin + testRestClient.getAndSetUserToken(tenantAdminId); + + CalculatedField savedCalculatedField = createScriptCalculatedField(deviceProfileId); + + await().alias("create CF -> perform initial calculation for device by profile").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + JsonNode airDensity = testRestClient.getLatestTelemetry(device.getId()); + assertThat(airDensity).isNotNull(); + assertThat(airDensity.get("airDensity").get(0).get("value").asText()).isEqualTo("1.05"); + }); + + testRestClient.postTelemetryAttribute(asset.getId(), DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"altitude\":1531}")); + + await().alias("create CF -> update telemetry for common entity").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + JsonNode airDensity = testRestClient.getLatestTelemetry(device.getId()); + assertThat(airDensity).isNotNull(); + assertThat(airDensity.get("airDensity").get(0).get("value").asText()).isEqualTo("0.99"); + }); + + testRestClient.deleteCalculatedFieldIfExists(savedCalculatedField.getId()); + } + private CalculatedField createSimpleCalculatedField() { return createSimpleCalculatedField(device.getId()); } @@ -323,19 +352,21 @@ public class CalculatedFieldTest extends AbstractContainerTest { calculatedField.setName("Air density" + RandomStringUtils.randomAlphabetic(5)); calculatedField.setDebugSettings(DebugSettings.all()); - SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + ScriptCalculatedFieldConfiguration config = new ScriptCalculatedFieldConfiguration(); Argument argument1 = new Argument(); argument1.setRefEntityId(asset.getId()); ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("altitude", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); argument1.setRefEntityKey(refEntityKey1); - config.setArguments(Map.of("altitude", argument1)); Argument argument2 = new Argument(); ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("temperatureInF", ArgumentType.TS_ROLLING, null); + argument2.setTimeWindow(30000L); + argument2.setLimit(5); argument2.setRefEntityKey(refEntityKey2); - config.setArguments(Map.of("temperature", argument2)); - config.setExpression("return {\"airDensity\": 5};"); + config.setArguments(Map.of("altitude", argument1, "temperature", argument2)); + + config.setExpression(exampleScript); Output output = new Output(); output.setType(OutputType.TIME_SERIES); diff --git a/pom.xml b/pom.xml index 72195b1e4f..b25c355934 100755 --- a/pom.xml +++ b/pom.xml @@ -42,6 +42,7 @@ 4.0.2 2.4.0-b180830.0359 4.0.5 + 10.1.39 3.2.12 3.2.12 3.2.12 @@ -88,7 +89,7 @@ 1.18.32 1.2.5 1.2.5 - 4.1.115.Final + 4.1.119.Final 2.0.65.Final 1.1.18 1.7.1 @@ -1147,6 +1148,21 @@ jaxb-runtime ${jaxb-runtime.version} + + org.apache.tomcat.embed + tomcat-embed-core + ${tomcat.version} + + + org.apache.tomcat.embed + tomcat-embed-el + ${tomcat.version} + + + org.apache.tomcat.embed + tomcat-embed-websocket + ${tomcat.version} + org.springframework.boot spring-boot-starter diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index eb568f9cc7..7ee5ab1496 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -120,14 +120,6 @@ org.bouncycastle bcpkix-jdk18on - - org.locationtech.spatial4j - spatial4j - - - org.locationtech.jts - jts-core - com.sun.mail jakarta.mail diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/AbstractGeofencingNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/AbstractGeofencingNode.java index 8025baeec7..ff6b3acb88 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/AbstractGeofencingNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/AbstractGeofencingNode.java @@ -21,6 +21,11 @@ import com.google.gson.JsonParser; import org.locationtech.spatial4j.context.jts.JtsSpatialContext; import org.locationtech.spatial4j.context.jts.JtsSpatialContextFactory; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.geo.Coordinates; +import org.thingsboard.common.util.geo.GeoUtil; +import org.thingsboard.common.util.geo.Perimeter; +import org.thingsboard.common.util.geo.PerimeterType; +import org.thingsboard.common.util.geo.RangeUnit; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; @@ -74,7 +79,7 @@ public abstract class AbstractGeofencingNode void; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 1cf44966fc..c5b256ea89 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -37,6 +37,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, filter, switchMap, tap } from 'rxjs/operators'; import { + ArgumentEntityType, ArgumentType, CalculatedField, CalculatedFieldEventArguments, @@ -60,7 +61,7 @@ import { DatePipe } from '@angular/common'; export class CalculatedFieldsTableConfig extends EntityTableConfig { readonly calculatedFieldsDebugPerTenantLimitsConfiguration = - getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; + getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration']; readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; readonly tenantId = getCurrentAuthUser(this.store).tenantId; additionalDebugActionConfig = { @@ -159,7 +160,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.getCalculatedFieldDialog(calculatedField, 'action.add')), + switchMap(calculatedField => this.getCalculatedFieldDialog(this.updateImportedCalculatedField(calculatedField), 'action.add', true)), filter(Boolean), switchMap(calculatedField => this.calculatedFieldsService.saveCalculatedField(calculatedField)), filter(Boolean), @@ -261,6 +262,18 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } + private updateImportedCalculatedField(calculatedField: CalculatedField): CalculatedField { + calculatedField.configuration.arguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { + const arg = calculatedField.configuration.arguments[key]; + acc[key] = arg.refEntityId?.entityType === ArgumentEntityType.Tenant + ? { ...arg, refEntityId: { id: this.tenantId, entityType: ArgumentEntityType.Tenant } } + : arg; + return acc; + }, {}); + + return calculatedField; + } + private onDebugConfigChanged(id: string, debugSettings: EntityDebugSettings): void { this.calculatedFieldsService.getCalculatedFieldById(id).pipe( switchMap(field => this.calculatedFieldsService.saveCalculatedField({ ...field, debugSettings })), diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index abb34cb502..53d8a9d7b6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -58,7 +58,7 @@ - @if (argument.refEntityId?.id) { + @if (argument.refEntityId?.id && argument.refEntityId?.entityType !== ArgumentEntityType.Tenant) { {{ entityNameMap.get(argument.refEntityId.id) ?? '' }} @@ -87,17 +87,17 @@ - +