diff --git a/application/src/main/data/json/system/scada_symbols/3-phase-voltage-relay-hp.svg b/application/src/main/data/json/system/scada_symbols/3-phase-voltage-relay-hp.svg index 8e1a46978e..6107f8f212 100644 --- a/application/src/main/data/json/system/scada_symbols/3-phase-voltage-relay-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/3-phase-voltage-relay-hp.svg @@ -535,7 +535,7 @@ } ] }]]> -220220220v +220220220v diff --git a/application/src/main/data/json/system/scada_symbols/battery-hp.svg b/application/src/main/data/json/system/scada_symbols/battery-hp.svg index 8b63a9ab29..3ed3acfe01 100644 --- a/application/src/main/data/json/system/scada_symbols/battery-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/battery-hp.svg @@ -459,7 +459,7 @@ - ON + ON diff --git a/application/src/main/data/json/system/scada_symbols/conical-tank.svg b/application/src/main/data/json/system/scada_symbols/conical-tank.svg index 592508ee30..59988fa963 100644 --- a/application/src/main/data/json/system/scada_symbols/conical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/conical-tank.svg @@ -267,7 +267,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/control-panel-hp.svg b/application/src/main/data/json/system/scada_symbols/control-panel-hp.svg index d9b0857156..6fb7d3a501 100644 --- a/application/src/main/data/json/system/scada_symbols/control-panel-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/control-panel-hp.svg @@ -320,13 +320,13 @@ } ] }]]> -Heat pump +Heat pump - On + On - Off + Off \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg index 5ac0af8248..926f9bc33f 100644 --- a/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg @@ -489,14 +489,13 @@ "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, - "disabled": false + "disabled": false, + "visible": true }, { "id": "lineColor", @@ -584,4 +583,4 @@ ] }]]> - \ No newline at end of file + diff --git a/application/src/main/data/json/system/scada_symbols/curcuit-breaker-hp.svg b/application/src/main/data/json/system/scada_symbols/curcuit-breaker-hp.svg index 12f8b4944f..9aca99fcd4 100644 --- a/application/src/main/data/json/system/scada_symbols/curcuit-breaker-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/curcuit-breaker-hp.svg @@ -425,7 +425,7 @@ - ON + ON diff --git a/application/src/main/data/json/system/scada_symbols/cylindrical-tank.svg b/application/src/main/data/json/system/scada_symbols/cylindrical-tank.svg index 5675818fe7..231bc83efe 100644 --- a/application/src/main/data/json/system/scada_symbols/cylindrical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/cylindrical-tank.svg @@ -563,7 +563,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/dynamic-horizontal-scale-hp.svg b/application/src/main/data/json/system/scada_symbols/dynamic-horizontal-scale-hp.svg index 7b11968223..3e68be83c3 100644 --- a/application/src/main/data/json/system/scada_symbols/dynamic-horizontal-scale-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/dynamic-horizontal-scale-hp.svg @@ -586,13 +586,13 @@ } ] }]]> -Outdoor°C +Outdoor°C 0 100 - 26 + 26 diff --git a/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg b/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg index 37ba3df7c2..779fc069df 100644 --- a/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/dynamic-vertical-scale-hp.svg @@ -579,19 +579,19 @@ } ] }]]> -Outdoor°C +Outdoor°C - + 100 0 - 26 + 26 diff --git a/application/src/main/data/json/system/scada_symbols/elevated-tank.svg b/application/src/main/data/json/system/scada_symbols/elevated-tank.svg index 0d82a10b41..761f9c1642 100644 --- a/application/src/main/data/json/system/scada_symbols/elevated-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/elevated-tank.svg @@ -557,7 +557,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/energy-meter-hp.svg b/application/src/main/data/json/system/scada_symbols/energy-meter-hp.svg index 3855d66ec8..0eb47a923f 100644 --- a/application/src/main/data/json/system/scada_symbols/energy-meter-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/energy-meter-hp.svg @@ -474,7 +474,7 @@ } ] }]]> -000023kWhT1 +000023kWhT1 diff --git a/application/src/main/data/json/system/scada_symbols/four-rate-energy-meter-hp.svg b/application/src/main/data/json/system/scada_symbols/four-rate-energy-meter-hp.svg index e095c1417f..2701e46684 100644 --- a/application/src/main/data/json/system/scada_symbols/four-rate-energy-meter-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/four-rate-energy-meter-hp.svg @@ -877,7 +877,7 @@ } ] }]]> -T1T2T3Export000223000223000223000223kWh +T1T2T3Export000223000223000223000223kWh diff --git a/application/src/main/data/json/system/scada_symbols/heat-pump-hp.svg b/application/src/main/data/json/system/scada_symbols/heat-pump-hp.svg index b1e643698c..f8fab04123 100644 --- a/application/src/main/data/json/system/scada_symbols/heat-pump-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/heat-pump-hp.svg @@ -489,7 +489,7 @@ - 27 + 27 diff --git a/application/src/main/data/json/system/scada_symbols/horizontal-curcuit-breaker-hp.svg b/application/src/main/data/json/system/scada_symbols/horizontal-curcuit-breaker-hp.svg index 17fd1ba69e..fc4456e377 100644 --- a/application/src/main/data/json/system/scada_symbols/horizontal-curcuit-breaker-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/horizontal-curcuit-breaker-hp.svg @@ -423,7 +423,7 @@ }]]> - ON + ON diff --git a/application/src/main/data/json/system/scada_symbols/horizontal-energy-system-controller-hp.svg b/application/src/main/data/json/system/scada_symbols/horizontal-energy-system-controller-hp.svg index 13ee85821b..7c125465b4 100644 --- a/application/src/main/data/json/system/scada_symbols/horizontal-energy-system-controller-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/horizontal-energy-system-controller-hp.svg @@ -364,7 +364,7 @@ } ] }]]> -Connected +Connected diff --git a/application/src/main/data/json/system/scada_symbols/horizontal-tank.svg b/application/src/main/data/json/system/scada_symbols/horizontal-tank.svg index c02da77258..99dde101de 100644 --- a/application/src/main/data/json/system/scada_symbols/horizontal-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/horizontal-tank.svg @@ -572,7 +572,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/large-conical-tank.svg b/application/src/main/data/json/system/scada_symbols/large-conical-tank.svg index 6789f8f7c9..ea9405da5f 100644 --- a/application/src/main/data/json/system/scada_symbols/large-conical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/large-conical-tank.svg @@ -268,7 +268,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/large-cylindrical-tank.svg b/application/src/main/data/json/system/scada_symbols/large-cylindrical-tank.svg index c9d9361d7d..73e31b4798 100644 --- a/application/src/main/data/json/system/scada_symbols/large-cylindrical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/large-cylindrical-tank.svg @@ -563,7 +563,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/large-stand-cylindrical-tank.svg b/application/src/main/data/json/system/scada_symbols/large-stand-cylindrical-tank.svg index 8e5c057209..f9af0f9c60 100644 --- a/application/src/main/data/json/system/scada_symbols/large-stand-cylindrical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/large-stand-cylindrical-tank.svg @@ -564,7 +564,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/large-stand-vertical-tank.svg b/application/src/main/data/json/system/scada_symbols/large-stand-vertical-tank.svg index 9b6763e0ea..508b83088a 100644 --- a/application/src/main/data/json/system/scada_symbols/large-stand-vertical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/large-stand-vertical-tank.svg @@ -564,7 +564,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/large-vertical-tank.svg b/application/src/main/data/json/system/scada_symbols/large-vertical-tank.svg index 75ff5ef979..d8283a8d29 100644 --- a/application/src/main/data/json/system/scada_symbols/large-vertical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/large-vertical-tank.svg @@ -563,7 +563,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/left-analog-water-level-meter.svg b/application/src/main/data/json/system/scada_symbols/left-analog-water-level-meter.svg index 2cdc9e587a..89c8b64a08 100644 --- a/application/src/main/data/json/system/scada_symbols/left-analog-water-level-meter.svg +++ b/application/src/main/data/json/system/scada_symbols/left-analog-water-level-meter.svg @@ -679,7 +679,7 @@ }]]> - Water + Water diff --git a/application/src/main/data/json/system/scada_symbols/left-heat-pump.svg b/application/src/main/data/json/system/scada_symbols/left-heat-pump.svg index 6232f3dda3..f7f45f50fd 100644 --- a/application/src/main/data/json/system/scada_symbols/left-heat-pump.svg +++ b/application/src/main/data/json/system/scada_symbols/left-heat-pump.svg @@ -584,7 +584,7 @@ - 27 + 27 diff --git a/application/src/main/data/json/system/scada_symbols/meter.svg b/application/src/main/data/json/system/scada_symbols/meter.svg index 6fdd27d2fa..f426e49747 100644 --- a/application/src/main/data/json/system/scada_symbols/meter.svg +++ b/application/src/main/data/json/system/scada_symbols/meter.svg @@ -720,7 +720,7 @@ - 37% + 37% diff --git a/application/src/main/data/json/system/scada_symbols/pool.svg b/application/src/main/data/json/system/scada_symbols/pool.svg index 75f21e9457..b6f03a7b6d 100644 --- a/application/src/main/data/json/system/scada_symbols/pool.svg +++ b/application/src/main/data/json/system/scada_symbols/pool.svg @@ -232,7 +232,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/right-analog-water-level-meter.svg b/application/src/main/data/json/system/scada_symbols/right-analog-water-level-meter.svg index 1b1be4f007..a66f39715b 100644 --- a/application/src/main/data/json/system/scada_symbols/right-analog-water-level-meter.svg +++ b/application/src/main/data/json/system/scada_symbols/right-analog-water-level-meter.svg @@ -679,7 +679,7 @@ }]]> - Water + Water diff --git a/application/src/main/data/json/system/scada_symbols/right-heat-pump.svg b/application/src/main/data/json/system/scada_symbols/right-heat-pump.svg index a0802c6e66..954e32f36c 100644 --- a/application/src/main/data/json/system/scada_symbols/right-heat-pump.svg +++ b/application/src/main/data/json/system/scada_symbols/right-heat-pump.svg @@ -584,7 +584,7 @@ - 27 + 27 diff --git a/application/src/main/data/json/system/scada_symbols/sand-filter-hp.svg b/application/src/main/data/json/system/scada_symbols/sand-filter-hp.svg index 773837608e..0d99ba911e 100644 --- a/application/src/main/data/json/system/scada_symbols/sand-filter-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/sand-filter-hp.svg @@ -621,32 +621,32 @@ - Filter + Filter - Backwash + Backwash - Rinse + Rinse - Waste + Waste - Recirculate + Recirculate - Closed + Closed \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/sand-filter.svg b/application/src/main/data/json/system/scada_symbols/sand-filter.svg index 243d5ed6e8..c0f6b8b417 100644 --- a/application/src/main/data/json/system/scada_symbols/sand-filter.svg +++ b/application/src/main/data/json/system/scada_symbols/sand-filter.svg @@ -408,37 +408,37 @@ - Filter + Filter - Backwash + Backwash - Rinse + Rinse - Waste + Waste - Recirculate + Recirculate - Closed + Closed diff --git a/application/src/main/data/json/system/scada_symbols/simple-horizontal-scale-hp.svg b/application/src/main/data/json/system/scada_symbols/simple-horizontal-scale-hp.svg index 9425ed48f2..76e3bc9eef 100644 --- a/application/src/main/data/json/system/scada_symbols/simple-horizontal-scale-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/simple-horizontal-scale-hp.svg @@ -490,13 +490,13 @@ } ] }]]> -Outdoor°C +Outdoor°C 0 100 - 26 + 26 diff --git a/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg b/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg index 4e14130ef3..c6fb05fd26 100644 --- a/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/simple-vertical-scale-hp.svg @@ -490,19 +490,19 @@ } ] }]]> -Outdoor°C +Outdoor°C - + 100 0 - 26 + 26 diff --git a/application/src/main/data/json/system/scada_symbols/small-cylindrical-tank.svg b/application/src/main/data/json/system/scada_symbols/small-cylindrical-tank.svg index 95c4fd3eb5..fda5351233 100644 --- a/application/src/main/data/json/system/scada_symbols/small-cylindrical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/small-cylindrical-tank.svg @@ -534,7 +534,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/small-left-meter.svg b/application/src/main/data/json/system/scada_symbols/small-left-meter.svg index 3d5d6fdcb9..1c70a5ee27 100644 --- a/application/src/main/data/json/system/scada_symbols/small-left-meter.svg +++ b/application/src/main/data/json/system/scada_symbols/small-left-meter.svg @@ -720,6 +720,6 @@ - 37% + 37% \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/small-meter.svg b/application/src/main/data/json/system/scada_symbols/small-meter.svg index a639475227..d66d70e048 100644 --- a/application/src/main/data/json/system/scada_symbols/small-meter.svg +++ b/application/src/main/data/json/system/scada_symbols/small-meter.svg @@ -657,7 +657,7 @@ - 37% + 37% diff --git a/application/src/main/data/json/system/scada_symbols/small-right-center.svg b/application/src/main/data/json/system/scada_symbols/small-right-center.svg index 8afe7bc88e..d2f96e7848 100644 --- a/application/src/main/data/json/system/scada_symbols/small-right-center.svg +++ b/application/src/main/data/json/system/scada_symbols/small-right-center.svg @@ -669,7 +669,7 @@ - 37% + 37% diff --git a/application/src/main/data/json/system/scada_symbols/small-spherical-tank.svg b/application/src/main/data/json/system/scada_symbols/small-spherical-tank.svg index 8b0a1a20b0..ae0dea7fca 100644 --- a/application/src/main/data/json/system/scada_symbols/small-spherical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/small-spherical-tank.svg @@ -539,7 +539,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/spherical-tank.svg b/application/src/main/data/json/system/scada_symbols/spherical-tank.svg index 44cd98e6f9..8be8a6f0d5 100644 --- a/application/src/main/data/json/system/scada_symbols/spherical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/spherical-tank.svg @@ -569,7 +569,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/stand-cylindrical-tank.svg b/application/src/main/data/json/system/scada_symbols/stand-cylindrical-tank.svg index 9666f987d7..ef07d408e2 100644 --- a/application/src/main/data/json/system/scada_symbols/stand-cylindrical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/stand-cylindrical-tank.svg @@ -564,7 +564,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/stand-horizontal-tank.svg b/application/src/main/data/json/system/scada_symbols/stand-horizontal-tank.svg index 03995acbd5..cdb6885b3f 100644 --- a/application/src/main/data/json/system/scada_symbols/stand-horizontal-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/stand-horizontal-tank.svg @@ -573,7 +573,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/stand-vertical-short-tank.svg b/application/src/main/data/json/system/scada_symbols/stand-vertical-short-tank.svg index b448d24463..611aecec4d 100644 --- a/application/src/main/data/json/system/scada_symbols/stand-vertical-short-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/stand-vertical-short-tank.svg @@ -537,7 +537,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/stand-vertical-tank.svg b/application/src/main/data/json/system/scada_symbols/stand-vertical-tank.svg index c4dcc662fc..dd665214ca 100644 --- a/application/src/main/data/json/system/scada_symbols/stand-vertical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/stand-vertical-tank.svg @@ -566,7 +566,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/three-rate-energy-meter-hp.svg b/application/src/main/data/json/system/scada_symbols/three-rate-energy-meter-hp.svg index 526f6aa719..b35fe93c04 100644 --- a/application/src/main/data/json/system/scada_symbols/three-rate-energy-meter-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/three-rate-energy-meter-hp.svg @@ -745,7 +745,7 @@ } ] }]]> -T1T2T3000223000223000223kWh +T1T2T3000223000223000223kWh diff --git a/application/src/main/data/json/system/scada_symbols/two-rate-energy-meter-hp.svg b/application/src/main/data/json/system/scada_symbols/two-rate-energy-meter-hp.svg index e87548f059..325972b596 100644 --- a/application/src/main/data/json/system/scada_symbols/two-rate-energy-meter-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/two-rate-energy-meter-hp.svg @@ -613,7 +613,7 @@ } ] }]]> -T1T2000023000023kWh +T1T2000023000023kWh diff --git a/application/src/main/data/json/system/scada_symbols/vertical-energy-system-controller-hp.svg b/application/src/main/data/json/system/scada_symbols/vertical-energy-system-controller-hp.svg index 6da68556a2..15edf756bd 100644 --- a/application/src/main/data/json/system/scada_symbols/vertical-energy-system-controller-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/vertical-energy-system-controller-hp.svg @@ -364,7 +364,7 @@ } ] }]]> -Connected +Connected diff --git a/application/src/main/data/json/system/scada_symbols/vertical-short-tank.svg b/application/src/main/data/json/system/scada_symbols/vertical-short-tank.svg index 5d8aa42ab5..86a7ceef05 100644 --- a/application/src/main/data/json/system/scada_symbols/vertical-short-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/vertical-short-tank.svg @@ -536,7 +536,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/vertical-tank.svg b/application/src/main/data/json/system/scada_symbols/vertical-tank.svg index 5e51330ded..66c1cab666 100644 --- a/application/src/main/data/json/system/scada_symbols/vertical-tank.svg +++ b/application/src/main/data/json/system/scada_symbols/vertical-tank.svg @@ -566,7 +566,7 @@ - 1660 gal + 1660 gal diff --git a/application/src/main/data/json/system/scada_symbols/voltage-relay-hp.svg b/application/src/main/data/json/system/scada_symbols/voltage-relay-hp.svg index fa27214864..c943eaf15c 100644 --- a/application/src/main/data/json/system/scada_symbols/voltage-relay-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/voltage-relay-hp.svg @@ -426,7 +426,7 @@ } ] }]]> -220v +220v diff --git a/application/src/main/data/json/system/scada_symbols/voltage-stabilizer-hp.svg b/application/src/main/data/json/system/scada_symbols/voltage-stabilizer-hp.svg index 2ccad581d4..c3bba6ad4d 100644 --- a/application/src/main/data/json/system/scada_symbols/voltage-stabilizer-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/voltage-stabilizer-hp.svg @@ -570,7 +570,7 @@ } ] }]]> -220230inout +220230inout diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index f5033cce80..c143214e2b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -270,10 +270,19 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso rpc.setExpirationTime(request.getExpirationTime()); rpc.setRequest(JacksonUtil.valueToTree(request)); rpc.setStatus(status); - rpc.setAdditionalInfo(JacksonUtil.toJsonNode(request.getAdditionalInfo())); + rpc.setAdditionalInfo(getAdditionalInfo(request)); systemContext.getTbRpcService().save(tenantId, rpc); } + private JsonNode getAdditionalInfo(ToDeviceRpcRequest request) { + try { + return JacksonUtil.toJsonNode(request.getAdditionalInfo()); + } catch (IllegalArgumentException e) { + log.debug("Failed to parse additional info [{}]", request.getAdditionalInfo()); + return JacksonUtil.valueToTree(request.getAdditionalInfo()); + } + } + private ToDeviceRpcRequestMsg createToDeviceRpcRequestMsg(ToDeviceRpcRequest request) { ToDeviceRpcRequestBody body = request.getBody(); return ToDeviceRpcRequestMsg.newBuilder() diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java index 1193f935a0..5c5eea32fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java @@ -38,6 +38,7 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.domain.DomainService; import org.thingsboard.server.dao.edge.EdgeEventService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.edge.stats.EdgeStatsCounterService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.notification.NotificationRuleService; import org.thingsboard.server.dao.notification.NotificationTargetService; @@ -78,6 +79,7 @@ import org.thingsboard.server.service.executors.GrpcCallbackExecutorService; import java.util.EnumMap; import java.util.List; import java.util.Map; +import java.util.Optional; @Lazy @Data @@ -198,6 +200,9 @@ public class EdgeContextComponent { @Autowired private WidgetsBundleService widgetsBundleService; + @Autowired + private Optional statsCounterService; + // processors @Autowired diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index eaef1f7c7d..7340696788 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -59,8 +59,7 @@ import org.thingsboard.server.gen.edge.v1.RequestMsg; import org.thingsboard.server.gen.edge.v1.ResponseMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.kafka.KafkaAdmin; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -153,10 +152,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private TbCoreQueueFactory tbCoreQueueFactory; @Autowired - private Optional kafkaSettings; - - @Autowired - private Optional kafkaTopicConfigs; + private Optional kafkaAdmin; private Server server; @@ -232,8 +228,8 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } private EdgeGrpcSession createEdgeGrpcSession(StreamObserver outputStream) { - return kafkaSettings.isPresent() && kafkaTopicConfigs.isPresent() - ? new KafkaEdgeGrpcSession(ctx, topicService, tbCoreQueueFactory, kafkaSettings.get(), kafkaTopicConfigs.get(), outputStream, this::onEdgeConnect, this::onEdgeDisconnect, + return kafkaAdmin.isPresent() + ? new KafkaEdgeGrpcSession(ctx, topicService, tbCoreQueueFactory, kafkaAdmin.get(), outputStream, this::onEdgeConnect, this::onEdgeDisconnect, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession) : new PostgresEdgeGrpcSession(ctx, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); @@ -643,10 +639,10 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i List toRemove = new ArrayList<>(); for (EdgeGrpcSession session : sessions.values()) { if (session instanceof KafkaEdgeGrpcSession kafkaSession && - !kafkaSession.isConnected() && - kafkaSession.getConsumer() != null && - kafkaSession.getConsumer().getConsumer() != null && - !kafkaSession.getConsumer().getConsumer().isStopped()) { + !kafkaSession.isConnected() && + kafkaSession.getConsumer() != null && + kafkaSession.getConsumer().getConsumer() != null && + !kafkaSession.getConsumer().getConsumer().isStopped()) { toRemove.add(kafkaSession.getEdge().getId()); } } @@ -663,4 +659,5 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i log.warn("Failed to cleanup kafka sessions", e); } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index be65dd13a1..521730741f 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; +import org.thingsboard.server.dao.edge.stats.EdgeStatsKey; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; @@ -298,6 +299,10 @@ public abstract class EdgeGrpcSession implements Closeable { processHighPriorityEvents(); PageData pageData = fetcher.fetchEdgeEvents(edge.getTenantId(), edge, pageLink); if (isConnected() && !pageData.getData().isEmpty()) { + if (fetcher instanceof GeneralEdgeEventFetcher) { + long queueSize = pageData.getTotalElements() - ((long) pageLink.getPageSize() * pageLink.getPage()); + ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.setDownlinkMsgsLag(edge.getTenantId(), edge.getId(), queueSize)); + } log.trace("[{}][{}][{}] event(s) are going to be processed.", tenantId, edge.getId(), pageData.getData().size()); List downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData()); Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<>() { @@ -461,6 +466,8 @@ public abstract class EdgeGrpcSession implements Closeable { ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(error).build()); } + ctx.getStatsCounterService().ifPresent(statsCounterService -> + statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_TMP_FAILED, edge.getTenantId(), edge.getId(), 1)); log.warn("[{}][{}] {} on attempt {}", tenantId, edge.getId(), failureMsg, attempt); log.debug("[{}][{}] entities in failed batch: {}", tenantId, edge.getId(), copy); } @@ -474,6 +481,8 @@ public abstract class EdgeGrpcSession implements Closeable { log.error("[{}][{}][{}] {} Message {}", tenantId, edge.getId(), sessionId, message, downlinkMsg); ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(message).error(error).build()); + ctx.getStatsCounterService().ifPresent(statsCounterService -> + statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED, edge.getTenantId(), edge.getId(), 1)); sessionState.getPendingMsgsMap().remove(downlinkMsg.getDownlinkMsgId()); } else { sendDownlinkMsg(ResponseMsg.newBuilder() @@ -490,6 +499,7 @@ public abstract class EdgeGrpcSession implements Closeable { ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg) .error("Failed to deliver messages after " + MAX_DOWNLINK_ATTEMPTS + " attempts").build()); + ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED, edge.getTenantId(), edge.getId(), copy.size())); stopCurrentSendDownlinkMsgsTask(false); } } else { @@ -529,6 +539,7 @@ public abstract class EdgeGrpcSession implements Closeable { try { if (msg.getSuccess()) { sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId()); + ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PUSHED, edge.getTenantId(), edge.getId(), 1)); log.debug("[{}][{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg); } else { log.debug("[{}][{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg.getErrorMsg()); @@ -649,7 +660,8 @@ public abstract class EdgeGrpcSession implements Closeable { log.trace("[{}][{}] entity message processed [{}]", tenantId, edge.getId(), downlinkMsg); } } - case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); + case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> + downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); default -> log.warn("[{}][{}] Unsupported action type [{}]", tenantId, edge.getId(), edgeEvent.getAction()); } } catch (Exception e) { @@ -795,6 +807,7 @@ public abstract class EdgeGrpcSession implements Closeable { } } highPriorityQueue.add(edgeEvent); + ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edge.getTenantId(), edgeEvent.getEdgeId(), 1)); } protected ListenableFuture> processUplinkMsg(UplinkMsg uplinkMsg) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java index 8c3d897c1e..bc00ef4481 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java @@ -25,11 +25,14 @@ import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.BaseEdgeEventService; +import org.thingsboard.server.dao.edge.stats.EdgeStatsCounterService; +import org.thingsboard.server.dao.edge.stats.EdgeStatsKey; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import java.util.Optional; import java.util.UUID; @Slf4j @@ -40,6 +43,7 @@ public class KafkaEdgeEventService extends BaseEdgeEventService { private final TopicService topicService; private final TbQueueProducerProvider producerProvider; + private final Optional statsCounterService; @Override public ListenableFuture saveAsync(EdgeEvent edgeEvent) { @@ -48,7 +52,7 @@ public class KafkaEdgeEventService extends BaseEdgeEventService { TopicPartitionInfo tpi = topicService.getEdgeEventNotificationsTopic(edgeEvent.getTenantId(), edgeEvent.getEdgeId()); ToEdgeEventNotificationMsg msg = ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build(); producerProvider.getTbEdgeEventsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); - + statsCounterService.ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1)); return Futures.immediateFuture(null); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index ab0b42abb4..d165be33d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -32,9 +32,7 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.kafka.KafkaAdmin; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.service.edge.EdgeContextComponent; @@ -51,9 +49,7 @@ public class KafkaEdgeGrpcSession extends EdgeGrpcSession { private final TopicService topicService; private final TbCoreQueueFactory tbCoreQueueFactory; - - private final TbKafkaSettings kafkaSettings; - private final TbKafkaTopicConfigs kafkaTopicConfigs; + private final KafkaAdmin kafkaAdmin; private volatile boolean isHighPriorityProcessing; @@ -63,21 +59,20 @@ public class KafkaEdgeGrpcSession extends EdgeGrpcSession { private ExecutorService consumerExecutor; public KafkaEdgeGrpcSession(EdgeContextComponent ctx, TopicService topicService, TbCoreQueueFactory tbCoreQueueFactory, - TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs kafkaTopicConfigs, StreamObserver outputStream, + KafkaAdmin kafkaAdmin, StreamObserver outputStream, BiConsumer sessionOpenListener, BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { super(ctx, outputStream, sessionOpenListener, sessionCloseListener, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); this.topicService = topicService; this.tbCoreQueueFactory = tbCoreQueueFactory; - this.kafkaSettings = kafkaSettings; - this.kafkaTopicConfigs = kafkaTopicConfigs; + this.kafkaAdmin = kafkaAdmin; } private void processMsgs(List> msgs, TbQueueConsumer> consumer) { log.trace("[{}][{}] starting processing edge events", tenantId, edge.getId()); if (!isConnected() || isSyncInProgress() || isHighPriorityProcessing) { log.debug("[{}][{}] edge not connected, edge sync is not completed or high priority processing in progress, " + - "connected = {}, sync in progress = {}, high priority in progress = {}. Skipping iteration", + "connected = {}, sync in progress = {}, high priority in progress = {}. Skipping iteration", tenantId, edge.getId(), isConnected(), isSyncInProgress(), isHighPriorityProcessing); return; } @@ -159,7 +154,6 @@ public class KafkaEdgeGrpcSession extends EdgeGrpcSession { @Override public void cleanUp() { String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edge.getId()).getTopic(); - TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); kafkaAdmin.deleteTopic(topic); kafkaAdmin.deleteConsumerGroup(topic); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/stats/EdgeStatsService.java b/application/src/main/java/org/thingsboard/server/service/edge/stats/EdgeStatsService.java new file mode 100644 index 0000000000..48b2a47cfb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/stats/EdgeStatsService.java @@ -0,0 +1,150 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.stats; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.edge.stats.EdgeStatsCounterService; +import org.thingsboard.server.dao.edge.stats.MsgCounters; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.kafka.KafkaAdmin; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_ADDED; +import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_LAG; +import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED; +import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_PUSHED; +import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_TMP_FAILED; + +@TbCoreComponent +@ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true", matchIfMissing = false) +@RequiredArgsConstructor +@Service +@Slf4j +public class EdgeStatsService { + + private final TimeseriesService tsService; + private final EdgeStatsCounterService statsCounterService; + private final TopicService topicService; + private final Optional kafkaAdmin; + + @Value("${edges.stats.ttl:30}") + private int edgesStatsTtlDays; + @Value("${edges.stats.report-interval-millis:600000}") + private long reportIntervalMillis; + + + @Scheduled( + fixedDelayString = "${edges.stats.report-interval-millis:600000}", + initialDelayString = "${edges.stats.report-interval-millis:600000}" + ) + public void reportStats() { + log.debug("Reporting Edge communication stats..."); + long now = System.currentTimeMillis(); + long ts = now - (now % reportIntervalMillis); + + Map countersByEdge = statsCounterService.getCounterByEdge(); + Map lagByEdgeId = kafkaAdmin.isPresent() ? getEdgeLagByEdgeId(countersByEdge) : Collections.emptyMap(); + Map countersByEdgeSnapshot = new HashMap<>(statsCounterService.getCounterByEdge()); + countersByEdgeSnapshot.forEach((edgeId, counters) -> { + TenantId tenantId = counters.getTenantId(); + + if (kafkaAdmin.isPresent()) { + counters.getMsgsLag().set(lagByEdgeId.getOrDefault(edgeId, 0L)); + } + List statsEntries = List.of( + entry(ts, DOWNLINK_MSGS_ADDED.getKey(), counters.getMsgsAdded().get()), + entry(ts, DOWNLINK_MSGS_PUSHED.getKey(), counters.getMsgsPushed().get()), + entry(ts, DOWNLINK_MSGS_PERMANENTLY_FAILED.getKey(), counters.getMsgsPermanentlyFailed().get()), + entry(ts, DOWNLINK_MSGS_TMP_FAILED.getKey(), counters.getMsgsTmpFailed().get()), + entry(ts, DOWNLINK_MSGS_LAG.getKey(), counters.getMsgsLag().get()) + ); + + log.trace("Reported Edge communication stats: {} tenantId - {}, edgeId - {}", statsEntries, tenantId, edgeId); + saveTs(tenantId, edgeId, statsEntries); + }); + } + + private Map getEdgeLagByEdgeId(Map countersByEdge) { + Map edgeToTopicMap = countersByEdge.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> topicService.buildEdgeEventNotificationsTopicPartitionInfo(e.getValue().getTenantId(), e.getKey()).getTopic() + )); + + Map lagByTopic = kafkaAdmin.get().getTotalLagForGroupsBulk(new HashSet<>(edgeToTopicMap.values())); + + return edgeToTopicMap.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> lagByTopic.getOrDefault(e.getValue(), 0L) + )); + } + + private void saveTs(TenantId tenantId, EdgeId edgeId, List statsEntries) { + try { + ListenableFuture future = tsService.save( + tenantId, + edgeId, + statsEntries, + TimeUnit.DAYS.toSeconds(edgesStatsTtlDays) + ); + + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(TimeseriesSaveResult result) { + log.debug("Successfully saved edge time-series stats: {} for edge: {}", statsEntries, edgeId); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to save edge time-series stats for edge: {}", edgeId, t); + } + }, MoreExecutors.directExecutor()); + } finally { + statsCounterService.clear(edgeId); + } + } + + private BasicTsKvEntry entry(long ts, String key, long value) { + return new BasicTsKvEntry(ts, new LongDataEntry(key, value)); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java index 43b0c575a0..5ded4f66c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -20,10 +20,8 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsConfig; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.KafkaAdmin; -import java.util.Collections; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -33,8 +31,7 @@ public class KafkaEdqsSyncService extends EdqsSyncService { private final boolean syncNeeded; - public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings, TopicService topicService, EdqsConfig edqsConfig) { - TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap()); + public KafkaEdqsSyncService(KafkaAdmin kafkaAdmin, TopicService topicService, EdqsConfig edqsConfig) { this.syncNeeded = kafkaAdmin.areAllTopicsEmpty(IntStream.range(0, edqsConfig.getPartitions()) .mapToObj(partition -> TopicPartitionInfo.builder() .topic(topicService.buildTopicName(edqsConfig.getEventsTopic())) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index 0d4cc26ce7..26f0d81182 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -176,8 +176,8 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb for (int i = oldPartitions; i < newPartitions; i++) { tbQueueAdmin.createTopicIfNotExists( new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(), - queue.getCustomProperties() - ); + queue.getCustomProperties(), + true); // forcing topic creation because the topic may still be cached on some nodes } } diff --git a/application/src/main/java/org/thingsboard/server/service/notification/provider/DefaultFirebaseService.java b/application/src/main/java/org/thingsboard/server/service/notification/provider/DefaultFirebaseService.java index d1307fcd5d..8056b9fde2 100644 --- a/application/src/main/java/org/thingsboard/server/service/notification/provider/DefaultFirebaseService.java +++ b/application/src/main/java/org/thingsboard/server/service/notification/provider/DefaultFirebaseService.java @@ -140,8 +140,10 @@ public class DefaultFirebaseService implements FirebaseService { } public void destroy() { - app.delete(); - app = null; + if (app != null) { + app.delete(); + app = null; + } messaging = null; log.debug("[{}] Destroyed FirebaseContext", key); } diff --git a/application/src/main/java/org/thingsboard/server/service/notification/provider/DefaultSlackService.java b/application/src/main/java/org/thingsboard/server/service/notification/provider/DefaultSlackService.java index 85f0642c55..ec835d152f 100644 --- a/application/src/main/java/org/thingsboard/server/service/notification/provider/DefaultSlackService.java +++ b/application/src/main/java/org/thingsboard/server/service/notification/provider/DefaultSlackService.java @@ -15,20 +15,25 @@ */ package org.thingsboard.server.service.notification.provider; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.slack.api.Slack; import com.slack.api.methods.MethodsClient; import com.slack.api.methods.SlackApiRequest; import com.slack.api.methods.SlackApiTextResponse; +import com.slack.api.methods.SlackFilesUploadV2Exception; import com.slack.api.methods.request.chat.ChatPostMessageRequest; import com.slack.api.methods.request.conversations.ConversationsListRequest; +import com.slack.api.methods.request.conversations.ConversationsOpenRequest; +import com.slack.api.methods.request.files.FilesUploadV2Request; import com.slack.api.methods.request.users.UsersListRequest; import com.slack.api.methods.response.conversations.ConversationsListResponse; import com.slack.api.methods.response.users.UsersListResponse; import com.slack.api.model.ConversationType; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.notification.SlackService; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.notification.NotificationDeliveryMethod; @@ -36,6 +41,8 @@ import org.thingsboard.server.common.data.notification.settings.NotificationSett import org.thingsboard.server.common.data.notification.settings.SlackNotificationDeliveryMethodConfig; import org.thingsboard.server.common.data.notification.targets.slack.SlackConversation; import org.thingsboard.server.common.data.notification.targets.slack.SlackConversationType; +import org.thingsboard.server.common.data.notification.targets.slack.SlackFile; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.data.util.ThrowingBiFunction; import org.thingsboard.server.dao.notification.NotificationSettingsService; @@ -58,11 +65,40 @@ public class DefaultSlackService implements SlackService { @Override public void sendMessage(TenantId tenantId, String token, String conversationId, String message) { - ChatPostMessageRequest request = ChatPostMessageRequest.builder() - .channel(conversationId) - .text(message) - .build(); - sendRequest(token, request, MethodsClient::chatPostMessage); + sendMessage(tenantId, token, conversationId, message, null); + } + + @Override + public void sendMessage(TenantId tenantId, String token, String conversationId, String message, List files) { + if (CollectionsUtil.isNotEmpty(files)) { + if (conversationId.startsWith("U")) { // direct message + /* + * files.uploadV2 requires an existing channel ID, while chat.postMessage auto‑opens DMs + * */ + conversationId = sendRequest(token, ConversationsOpenRequest.builder() + .users(List.of(conversationId)) + .build(), MethodsClient::conversationsOpen).getChannel().getId(); + } + + FilesUploadV2Request request = FilesUploadV2Request.builder() + .initialComment(message) + .channel(conversationId) + .uploadFiles(files.stream() + .map(file -> FilesUploadV2Request.UploadFile.builder() + .filename(file.getName()) + .title(file.getName()) + .fileData(file.getData()) + .build()) + .toList()) + .build(); + sendRequest(token, request, MethodsClient::filesUploadV2); + } else { + ChatPostMessageRequest request = ChatPostMessageRequest.builder() + .channel(conversationId) + .text(message) + .build(); + sendRequest(token, request, MethodsClient::chatPostMessage); + } } @Override @@ -128,22 +164,52 @@ public class DefaultSlackService implements SlackService { R response; try { response = method.apply(client, request); + } catch (SlackFilesUploadV2Exception e) { + if (e.getGetURLResponses() != null) { + e.getGetURLResponses().forEach(this::checkResponse); + } + if (e.getCompleteResponse() != null) { + checkResponse(e.getCompleteResponse()); + } + if (e.getFileInfoResponses() != null) { + e.getFileInfoResponses().forEach(this::checkResponse); + } + throw new RuntimeException("Failed to upload Slack file: " + e.toString(), e); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } - if (!response.isOk()) { - String error = response.getError(); - if (error == null) { - error = "unknown error"; - } else if (error.contains("missing_scope")) { - String neededScope = response.getNeeded(); - error = "bot token scope '" + neededScope + "' is needed"; - } - throw new RuntimeException("Slack API error: " + error); + checkResponse(response); + return response; + } + + + private void checkResponse(SlackApiTextResponse response) { + if (response.isOk()) { + return; } - return response; + String error = response.getError(); + if (error != null) { + switch (error) { + case "missing_scope" -> { + String neededScope = response.getNeeded(); + error = "bot token scope '" + neededScope + "' is needed"; + } + case "not_in_channel" -> { + error = "app needs to be added to the channel"; + } + default -> { + error = null; + } + } + } + if (error == null) { + ObjectNode responseJson = (ObjectNode) JacksonUtil.valueToTree(response); + responseJson.remove("httpResponseHeaders"); + error = responseJson.toString(); + } + throw new RuntimeException("Slack API error: " + error); } } diff --git a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java index b3fe76f154..45a73072f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java @@ -57,6 +57,7 @@ import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.sql.query.EntityKeyMapping; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -224,7 +225,7 @@ public class DefaultEntityQueryService implements EntityQueryService { private EntityDataQuery buildEntityDataQuery(AlarmCountQuery query) { EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null, - new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY))); + new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, EntityKeyMapping.CREATED_TIME))); return new EntityDataQuery(query.getEntityFilter(), edpl, null, null, query.getKeyFilters()); } @@ -232,7 +233,7 @@ public class DefaultEntityQueryService implements EntityQueryService { EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder(); EntityDataSortOrder entitiesSortOrder; if (sortOrder == null || sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { - entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY)); + entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, EntityKeyMapping.CREATED_TIME)); } else { entitiesSortOrder = sortOrder; } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java index 6d06c85585..3354c63f9f 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java @@ -30,9 +30,7 @@ import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.kafka.KafkaAdmin; import org.thingsboard.server.queue.util.TbCoreComponent; import java.time.Instant; @@ -57,7 +55,7 @@ public class KafkaEdgeTopicsCleanUpService extends AbstractCleanUpService { private final TenantService tenantService; private final EdgeService edgeService; private final AttributesService attributesService; - private final TbKafkaAdmin kafkaAdmin; + private final KafkaAdmin kafkaAdmin; @Value("${sql.ttl.edge_events.edge_events_ttl:2628000}") private long ttlSeconds; @@ -67,13 +65,13 @@ public class KafkaEdgeTopicsCleanUpService extends AbstractCleanUpService { public KafkaEdgeTopicsCleanUpService(PartitionService partitionService, EdgeService edgeService, TenantService tenantService, AttributesService attributesService, - TopicService topicService, TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { + TopicService topicService, KafkaAdmin kafkaAdmin) { super(partitionService); this.topicService = topicService; this.tenantService = tenantService; this.edgeService = edgeService; this.attributesService = attributesService; - this.kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.kafkaAdmin = kafkaAdmin; } @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.edge_events.execution_interval_ms})}", fixedDelayString = "${sql.ttl.edge_events.execution_interval_ms}") @@ -82,8 +80,8 @@ public class KafkaEdgeTopicsCleanUpService extends AbstractCleanUpService { return; } - Set topics = kafkaAdmin.getAllTopics(); - if (topics == null || topics.isEmpty()) { + Set topics = kafkaAdmin.listTopics(); + if (topics.isEmpty()) { return; } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e9cc7da571..c99163b833 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -742,9 +742,9 @@ redis: # Minumum number of idle connections that can be maintained in the pool without being closed minIdle: "${REDIS_POOL_CONFIG_MIN_IDLE:16}" # Enable/Disable PING command sent when a connection is borrowed - testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:true}" + testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:false}" # The property is used to specify whether to test the connection before returning it to the connection pool. - testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:true}" + testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:false}" # The property is used in the context of connection pooling in Redis testWhileIdle: "${REDIS_POOL_CONFIG_TEST_WHILE_IDLE:true}" # Minimum time that an idle connection should be idle before it can be evicted from the connection pool. The value is set in milliseconds @@ -1491,6 +1491,13 @@ edges: state: # Persist state of edge (active, last connect, last disconnect) into timeseries or attributes tables. 'false' means to store edge state into attributes table persistToTelemetry: "${EDGES_PERSIST_STATE_TO_TELEMETRY:false}" + stats: + # Enable or disable reporting of edge communication stats (true or false) + enabled: "${EDGES_STATS_ENABLED:true}" + # Time-to-live in days for stored edge communication stats in timeseries + ttl: "${EDGES_STATS_TTL:30}" + # How often to report edge communication stats in milliseconds + report-interval-millis: "${EDGES_STATS_REPORT_INTERVAL_MS:600000}" # Spring doc common parameters springdoc: diff --git a/application/src/test/java/org/thingsboard/server/service/edge/EdgeStatsTest.java b/application/src/test/java/org/thingsboard/server/service/edge/EdgeStatsTest.java new file mode 100644 index 0000000000..25ff0f1b5d --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/edge/EdgeStatsTest.java @@ -0,0 +1,173 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge; + +import com.google.common.util.concurrent.Futures; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.edge.stats.EdgeStatsCounterService; +import org.thingsboard.server.dao.edge.stats.MsgCounters; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.kafka.KafkaAdmin; +import org.thingsboard.server.service.edge.stats.EdgeStatsService; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_ADDED; +import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_LAG; +import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED; +import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_PUSHED; +import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_TMP_FAILED; + +@ExtendWith(MockitoExtension.class) +public class EdgeStatsTest { + + @Mock + private TimeseriesService tsService; + @Mock + private TopicService topicService; + @Mock + private EdgeStatsCounterService statsCounterService; + private EdgeStatsService edgeStatsService; + + private final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + private final EdgeId edgeId = new EdgeId(UUID.randomUUID()); + + @BeforeEach + void setUp() { + edgeStatsService = new EdgeStatsService( + tsService, + statsCounterService, + topicService, + Optional.empty() + ); + + ReflectionTestUtils.setField(edgeStatsService, "edgesStatsTtlDays", 30); + ReflectionTestUtils.setField(edgeStatsService, "reportIntervalMillis", 600_000L); + } + + @Test + public void testReportStatsSavesTelemetry() { + // given + MsgCounters counters = new MsgCounters(tenantId); + counters.getMsgsAdded().set(5); + counters.getMsgsPushed().set(3); + counters.getMsgsPermanentlyFailed().set(1); + counters.getMsgsTmpFailed().set(0); + counters.getMsgsLag().set(10); + + ConcurrentHashMap countersByEdge = new ConcurrentHashMap<>(); + countersByEdge.put(edgeId, counters); + + when(statsCounterService.getCounterByEdge()).thenReturn(countersByEdge); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + when(tsService.save(eq(tenantId), eq(edgeId), captor.capture(), anyLong())) + .thenReturn(Futures.immediateFuture(mock(TimeseriesSaveResult.class))); + + // when + edgeStatsService.reportStats(); + + // then + List entries = captor.getValue(); + Assertions.assertEquals(5, entries.size()); + + Map valuesByKey = entries.stream() + .collect(Collectors.toMap(TsKvEntry::getKey, e -> e.getLongValue().orElse(-1L))); + + Assertions.assertEquals(5L, valuesByKey.get(DOWNLINK_MSGS_ADDED.getKey()).longValue()); + Assertions.assertEquals(3L, valuesByKey.get(DOWNLINK_MSGS_PUSHED.getKey()).longValue()); + Assertions.assertEquals(1L, valuesByKey.get(DOWNLINK_MSGS_PERMANENTLY_FAILED.getKey()).longValue()); + Assertions.assertEquals(0L, valuesByKey.get(DOWNLINK_MSGS_TMP_FAILED.getKey()).longValue()); + Assertions.assertEquals(10L, valuesByKey.get(DOWNLINK_MSGS_LAG.getKey()).longValue()); + + + verify(statsCounterService).clear(edgeId); + } + + @Test + public void testReportStatsWithKafkaLag() { + // given + MsgCounters counters = new MsgCounters(tenantId); + counters.getMsgsAdded().set(2); + counters.getMsgsPushed().set(2); + counters.getMsgsPermanentlyFailed().set(0); + counters.getMsgsTmpFailed().set(1); + counters.getMsgsLag().set(0); + + ConcurrentHashMap countersByEdge = new ConcurrentHashMap<>(); + countersByEdge.put(edgeId, counters); + + // mocks + when(statsCounterService.getCounterByEdge()).thenReturn(countersByEdge); + + String topic = "edge-topic"; + TopicPartitionInfo partitionInfo = new TopicPartitionInfo(topic, tenantId, 0, false); + when(topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId)).thenReturn(partitionInfo); + + KafkaAdmin kafkaAdmin = mock(KafkaAdmin.class); + when(kafkaAdmin.getTotalLagForGroupsBulk(Set.of(topic))) + .thenReturn(Map.of(topic, 15L)); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + when(tsService.save(eq(tenantId), eq(edgeId), captor.capture(), anyLong())) + .thenReturn(Futures.immediateFuture(mock(TimeseriesSaveResult.class))); + + edgeStatsService = new EdgeStatsService( + tsService, + statsCounterService, + topicService, + Optional.of(kafkaAdmin) + ); + ReflectionTestUtils.setField(edgeStatsService, "edgesStatsTtlDays", 30); + ReflectionTestUtils.setField(edgeStatsService, "reportIntervalMillis", 600_000L); + + // when + edgeStatsService.reportStats(); + + // then + List entries = captor.getValue(); + Map valuesByKey = entries.stream() + .collect(Collectors.toMap(TsKvEntry::getKey, e -> e.getLongValue().orElse(-1L))); + + Assertions.assertEquals(15L, valuesByKey.get(DOWNLINK_MSGS_LAG.getKey())); + verify(statsCounterService).clear(edgeId); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 14f90822bf..8e95a68110 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -79,6 +79,7 @@ import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.data.query.StringFilterPredicate.StringOperation; +import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; @@ -118,8 +119,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; +import static org.thingsboard.server.common.data.AttributeScope.SERVER_SCOPE; import static org.thingsboard.server.common.data.query.EntityKeyType.ATTRIBUTE; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; +import static org.thingsboard.server.common.data.query.EntityKeyType.SERVER_ATTRIBUTE; @Slf4j @DaoSqlTest @@ -604,7 +607,7 @@ public class EntityServiceTest extends AbstractControllerTest { List> attributeFutures = new ArrayList<>(); for (int i = 0; i < assets.size(); i++) { Asset asset = assets.get(i); - attributeFutures.add(saveLongAttribute(asset.getId(), "consumption", consumptions.get(i), AttributeScope.SERVER_SCOPE)); + attributeFutures.add(saveLongAttribute(asset.getId(), "consumption", consumptions.get(i), SERVER_SCOPE)); } Futures.allAsList(attributeFutures).get(); @@ -1745,6 +1748,33 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } + @Test + public void testFindTenantTelemetry() { + // save timeseries by sys admin + BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, new DoubleDataEntry("temperature", 45.5)); + timeseriesService.save(TenantId.SYS_TENANT_ID, tenantId, timeseries); + + AttributeKvEntry attr = new BaseAttributeKvEntry(new LongDataEntry("attr", 10L), 42L); + attributesService.save(TenantId.SYS_TENANT_ID, tenantId, SERVER_SCOPE, List.of(attr)); + + SingleEntityFilter singleEntityFilter = new SingleEntityFilter(); + singleEntityFilter.setSingleEntity(AliasEntityId.fromEntityId(tenantId)); + + List entityFields = List.of( + new EntityKey(ENTITY_FIELD, "name") + ); + List latestValues = List.of( + new EntityKey(EntityKeyType.TIME_SERIES, "temperature"), + new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, "attr") + ); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, latestValues, null); + + findByQueryAndCheckTelemetry(query, EntityKeyType.TIME_SERIES, "temperature", List.of("45.5")); + findByQueryAndCheckTelemetry(query, EntityKeyType.SERVER_ATTRIBUTE, "attr", List.of("10")); + } + @Test public void testBuildStringPredicateQueryOperations() throws ExecutionException, InterruptedException { diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 09bd02e5a4..bcbe52b5c9 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -507,7 +507,7 @@ public class TbRuleEngineQueueConsumerManagerTest { consumerManager.delete(true); - await().atMost(2, TimeUnit.SECONDS) + await().atMost(5, TimeUnit.SECONDS) .untilAsserted(() -> { verify(ruleEngineMsgProducer).send(any(), any(), any()); }); diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java index 8e06913d2c..8f51afed66 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java @@ -79,10 +79,10 @@ public abstract class TBRedisCacheConfiguration { @Value("${redis.pool_config.minIdle:16}") private int minIdle; - @Value("${redis.pool_config.testOnBorrow:true}") + @Value("${redis.pool_config.testOnBorrow:false}") private boolean testOnBorrow; - @Value("${redis.pool_config.testOnReturn:true}") + @Value("${redis.pool_config.testOnReturn:false}") private boolean testOnReturn; @Value("${redis.pool_config.testWhileIdle:true}") diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbEdgeQueueAdmin.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbEdgeQueueAdmin.java index 9be50bb145..9210d4c1a8 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbEdgeQueueAdmin.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbEdgeQueueAdmin.java @@ -16,7 +16,7 @@ package org.thingsboard.server.queue; public interface TbEdgeQueueAdmin extends TbQueueAdmin { + void syncEdgeNotificationsOffsets(String fatGroupId, String newGroupId); - void deleteConsumerGroup(String consumerGroupId); } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java index 48d9b3c34f..0b9925765c 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java @@ -18,12 +18,13 @@ package org.thingsboard.server.queue; public interface TbQueueAdmin { default void createTopicIfNotExists(String topic) { - createTopicIfNotExists(topic, null); + createTopicIfNotExists(topic, null, false); } - void createTopicIfNotExists(String topic, String properties); + void createTopicIfNotExists(String topic, String properties, boolean force); void destroy(); void deleteTopic(String topic); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ai/model/AiModelConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/ai/model/AiModelConfig.java index 0a2b41a91f..bfaa29a6e3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ai/model/AiModelConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ai/model/AiModelConfig.java @@ -38,7 +38,7 @@ import org.thingsboard.server.common.data.ai.provider.OpenAiProviderConfig; @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, + include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "provider", visible = true ) diff --git a/ui-ngx/src/app/shared/components/string-items-list.component.scss b/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/slack/SlackFile.java similarity index 66% rename from ui-ngx/src/app/shared/components/string-items-list.component.scss rename to common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/slack/SlackFile.java index 726344037c..1be73221c8 100644 --- a/ui-ngx/src/app/shared/components/string-items-list.component.scss +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/slack/SlackFile.java @@ -13,10 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -.tb-string-items-list { - .mat-mdc-standard-chip { - .mdc-evolution-chip__cell--primary, .mat-mdc-chip-action-label { - overflow: hidden; - } - } +package org.thingsboard.server.common.data.notification.targets.slack; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class SlackFile { + + private final String name; + private final String type; // one of https://api.slack.com/types/file#file_types + private final byte[] data; + } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index 7e2e99e662..b34abe5363 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -43,7 +43,7 @@ import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.edqs.EdqsExecutors; import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; import org.thingsboard.server.queue.edqs.KafkaEdqsQueueFactory; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.KafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import java.util.HashMap; @@ -68,6 +68,7 @@ public class KafkaEdqsStateService implements EdqsStateService { private final EdqsExecutors edqsExecutors; private final EdqsMapper mapper; private final TopicService topicService; + private final KafkaAdmin kafkaAdmin; @Autowired @Lazy private EdqsProcessor edqsProcessor; @@ -86,7 +87,6 @@ public class KafkaEdqsStateService implements EdqsStateService { @Override public void init(PartitionedQueueConsumerManager> eventConsumer, List> otherConsumers) { versionsStore = new VersionsStore(config.getVersionsCacheTtl()); - TbKafkaAdmin queueAdmin = queueFactory.getEdqsQueueAdmin(); stateConsumer = PartitionedQueueConsumerManager.>create() .queueKey(new QueueKey(ServiceType.EDQS, config.getStateTopic())) .topic(topicService.buildTopicName(config.getStateTopic())) @@ -106,7 +106,7 @@ public class KafkaEdqsStateService implements EdqsStateService { consumer.commit(); }) .consumerCreator((config, tpi) -> queueFactory.createEdqsStateConsumer()) - .queueAdmin(queueAdmin) + .queueAdmin(queueFactory.getEdqsQueueAdmin()) .consumerExecutor(edqsExecutors.getConsumersExecutor()) .taskExecutor(edqsExecutors.getConsumerTaskExecutor()) .scheduler(edqsExecutors.getScheduler()) @@ -174,7 +174,7 @@ public class KafkaEdqsStateService implements EdqsStateService { // (because we need to be able to consume the same topic-partition by multiple instances) Map offsets = new HashMap<>(); try { - queueAdmin.getConsumerGroupOffsets(eventsToBackupKafkaConsumer.getGroupId()) + kafkaAdmin.getConsumerGroupOffsets(eventsToBackupKafkaConsumer.getGroupId()) .forEach((topicPartition, offsetAndMetadata) -> { offsets.put(topicPartition.topic(), offsetAndMetadata.offset()); }); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 6d010a7a0d..be407bd3db 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -529,8 +529,10 @@ public class ProtoUtils { .setRequestIdMSB(msg.getMsg().getId().getMostSignificantBits()) .setRequestIdLSB(msg.getMsg().getId().getLeastSignificantBits()) .setOneway(msg.getMsg().isOneway()) - .setPersisted(msg.getMsg().isPersisted()) - .setAdditionalInfo(msg.getMsg().getAdditionalInfo()); + .setPersisted(msg.getMsg().isPersisted()); + if (msg.getMsg().getAdditionalInfo() != null) { + builder.setAdditionalInfo(msg.getMsg().getAdditionalInfo()); + } if (msg.getMsg().getRetries() != null) { builder.setRetries(msg.getMsg().getRetries()); } @@ -555,7 +557,9 @@ public class ProtoUtils { toDeviceRpcRequestMsg.getOneway(), toDeviceRpcRequestMsg.getExpirationTime(), new ToDeviceRpcRequestBody(toDeviceRpcRequestMsg.getMethodName(), toDeviceRpcRequestMsg.getParams()), - toDeviceRpcRequestMsg.getPersisted(), toDeviceRpcRequestMsg.hasRetries() ? toDeviceRpcRequestMsg.getRetries() : null, toDeviceRpcRequestMsg.getAdditionalInfo()); + toDeviceRpcRequestMsg.getPersisted(), + toDeviceRpcRequestMsg.hasRetries() ? toDeviceRpcRequestMsg.getRetries() : null, + toDeviceRpcRequestMsg.hasAdditionalInfo() ? toDeviceRpcRequestMsg.getAdditionalInfo() : null); return new ToDeviceRpcRequestActorMsg(proto.getServiceId(), toDeviceRpcRequest); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 3870f17c5b..8372753df4 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -699,7 +699,7 @@ message ToDeviceRpcRequestMsg { bool oneway = 7; bool persisted = 8; optional int32 retries = 9; - string additionalInfo = 10; + optional string additionalInfo = 10; } message ToDeviceRpcResponseMsg { diff --git a/common/proto/src/test/java/org/thingsboard/server/common/util/ProtoUtilsTest.java b/common/proto/src/test/java/org/thingsboard/server/common/util/ProtoUtilsTest.java index 25788721dc..a46e489268 100644 --- a/common/proto/src/test/java/org/thingsboard/server/common/util/ProtoUtilsTest.java +++ b/common/proto/src/test/java/org/thingsboard/server/common/util/ProtoUtilsTest.java @@ -21,6 +21,9 @@ import org.jeasy.random.EasyRandomParameters; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.Device; @@ -56,6 +59,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; +import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.edge.EdgeHighPriorityMsg; import org.thingsboard.server.common.msg.edge.FromEdgeSyncResponse; @@ -279,4 +283,65 @@ class ProtoUtilsTest { assertThat(actual).as(String.format(description, entityName, entityName)).isEqualTo(expected); } + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {"{\"key\":\"value\"}"}) + void testRpcWithVariousAdditionalInfoToProtoAndBack(String additionalInfo) { + UUID requestId = UUID.fromString("93405c57-5787-46ff-806e-670bb60f49b6"); + String methodName = "reboot"; + String params = ""; + String serviceId = "serviceId"; + long expirationTime = System.currentTimeMillis(); + int retries = 3; + + ToDeviceRpcRequest request = new ToDeviceRpcRequest( + requestId, + tenantId, + deviceId, + false, + expirationTime, + new ToDeviceRpcRequestBody(methodName, params), + true, + retries, + additionalInfo + ); + ToDeviceRpcRequestActorMsg msg = new ToDeviceRpcRequestActorMsg(serviceId, request); + + // Serialize + TransportProtos.ToDeviceActorNotificationMsgProto toProto = ProtoUtils.toProto(msg); + assertThat(toProto).isNotNull(); + assertThat(toProto.hasToDeviceRpcRequestMsg()).isTrue(); + + TransportProtos.ToDeviceRpcRequestActorMsgProto toDeviceRpcRequestActorMsgProto = toProto.getToDeviceRpcRequestMsg(); + assertThat(toDeviceRpcRequestActorMsgProto.hasToDeviceRpcRequestMsg()).isTrue(); + + TransportProtos.ToDeviceRpcRequestMsg toDeviceRpcRequestMsg = toDeviceRpcRequestActorMsgProto.getToDeviceRpcRequestMsg(); + assertThat(toDeviceRpcRequestMsg.getRequestIdMSB()).isEqualTo(requestId.getMostSignificantBits()); + assertThat(toDeviceRpcRequestMsg.getRequestIdLSB()).isEqualTo(requestId.getLeastSignificantBits()); + assertThat(toDeviceRpcRequestMsg.getMethodName()).isEqualTo(methodName); + assertThat(toDeviceRpcRequestMsg.getParams()).isEqualTo(params); + assertThat(toDeviceRpcRequestMsg.getExpirationTime()).isEqualTo(expirationTime); + assertThat(toDeviceRpcRequestMsg.getOneway()).isFalse(); + assertThat(toDeviceRpcRequestMsg.getPersisted()).isTrue(); + assertThat(toDeviceRpcRequestMsg.getRetries()).isEqualTo(retries); + + if (additionalInfo != null) { + assertThat(toDeviceRpcRequestMsg.hasAdditionalInfo()).isTrue(); + assertThat(toDeviceRpcRequestMsg.getAdditionalInfo()).isEqualTo(additionalInfo); + } else { + assertThat(toDeviceRpcRequestMsg.hasAdditionalInfo()).isFalse(); + } + + // Deserialize + ToDeviceActorNotificationMsg fromProto = ProtoUtils.fromProto(toProto); + assertThat(fromProto).isNotNull(); + assertThat(fromProto).isInstanceOf(ToDeviceRpcRequestActorMsg.class); + ToDeviceRpcRequestActorMsg toDeviceRpcRequestActorMsg = (ToDeviceRpcRequestActorMsg) fromProto; + + assertThat(toDeviceRpcRequestActorMsg.getDeviceId()).isEqualTo(deviceId); + assertThat(toDeviceRpcRequestActorMsg.getTenantId()).isEqualTo(tenantId); + assertThat(toDeviceRpcRequestActorMsg.getServiceId()).isEqualTo(serviceId); + assertThat(toDeviceRpcRequestActorMsg.getMsg()).isEqualTo(request); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java index 6d78fa8b4f..ecb2a2b771 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java @@ -43,7 +43,7 @@ public class RuleEngineTbQueueAdminFactory { return new TbQueueAdmin() { @Override - public void createTopicIfNotExists(String topic, String properties) { + public void createTopicIfNotExists(String topic, String properties, boolean force) { } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaAdmin.java new file mode 100644 index 0000000000..6261e81497 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaAdmin.java @@ -0,0 +1,285 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import jakarta.annotation.PreDestroy; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.concurrent.ConcurrentException; +import org.apache.commons.lang3.concurrent.LazyInitializer; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.ListOffsetsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.OffsetSpec; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.TopicExistsException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.CachedValue; +import org.thingsboard.server.queue.util.TbKafkaComponent; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +@TbKafkaComponent +@Component +@Slf4j +public class KafkaAdmin { + + /* + * TODO: Get rid of per consumer/producer TbKafkaAdmin, + * use single KafkaAdmin instance that accepts topicConfigs. + * */ + + private final TbKafkaSettings settings; + + @Value("${queue.kafka.request.timeout.ms:30000}") + private int requestTimeoutMs; + @Value("${queue.kafka.topics_cache_ttl_ms:300000}") // 5 minutes by default + private int topicsCacheTtlMs; + + private final LazyInitializer adminClient; + private final CachedValue> topics; + + public KafkaAdmin(@Lazy TbKafkaSettings settings) { + this.settings = settings; + this.adminClient = LazyInitializer.builder() + .setInitializer(() -> AdminClient.create(settings.toAdminProps())) + .get(); + this.topics = new CachedValue<>(() -> { + Set topics = ConcurrentHashMap.newKeySet(); + topics.addAll(listTopics()); + return topics; + }, topicsCacheTtlMs); + } + + public void createTopicIfNotExists(String topic, Map properties, boolean force) { + Set topics = getTopics(); + if (!force && topics.contains(topic)) { + log.trace("Topic {} already present in cache", topic); + return; + } + + log.debug("Creating topic {} with properties {}", topic, properties); + String numPartitionsStr = properties.remove(TbKafkaTopicConfigs.NUM_PARTITIONS_SETTING); + int partitions = numPartitionsStr != null ? Integer.parseInt(numPartitionsStr) : 1; + NewTopic newTopic = new NewTopic(topic, partitions, settings.getReplicationFactor()).configs(properties); + + try { + getClient().createTopics(List.of(newTopic)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); + topics.add(topic); + } catch (ExecutionException ee) { + log.trace("Failed to create topic {} with properties {}", topic, properties, ee); + if (ee.getCause() instanceof TopicExistsException) { + //do nothing + } else { + log.warn("[{}] Failed to create topic", topic, ee); + throw new RuntimeException(ee); + } + } catch (Exception e) { + log.warn("[{}] Failed to create topic", topic, e); + throw new RuntimeException(e); + } + } + + public void deleteTopic(String topic) { + log.debug("Deleting topic {}", topic); + try { + getClient().deleteTopics(List.of(topic)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); + } catch (Exception e) { + log.error("Failed to delete kafka topic [{}].", topic, e); + } + } + + private Set getTopics() { + return topics.get(); + } + + public Set listTopics() { + try { + Set topics = getClient().listTopics().names().get(requestTimeoutMs, TimeUnit.MILLISECONDS); + log.trace("Listed topics: {}", topics); + return topics; + } catch (Exception e) { + log.error("Failed to get all topics.", e); + return Collections.emptySet(); + } + } + + public Map getTotalLagForGroupsBulk(Set groupIds) { + Map result = new HashMap<>(); + for (String groupId : groupIds) { + result.put(groupId, getTotalConsumerGroupLag(groupId)); + } + return result; + } + + public long getTotalConsumerGroupLag(String groupId) { + try { + Map committedOffsets = getConsumerGroupOffsets(groupId); + if (committedOffsets.isEmpty()) { + return 0L; + } + + Map latestOffsetsSpec = committedOffsets.keySet().stream() + .collect(Collectors.toMap(tp -> tp, tp -> OffsetSpec.latest())); + + Map endOffsets = + getClient().listOffsets(latestOffsetsSpec).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); + + return committedOffsets.entrySet().stream() + .mapToLong(entry -> { + TopicPartition tp = entry.getKey(); + long committed = entry.getValue().offset(); + long end = endOffsets.getOrDefault(tp, + new ListOffsetsResult.ListOffsetsResultInfo(0L, 0L, Optional.empty())).offset(); + return end - committed; + }).sum(); + + } catch (Exception e) { + log.error("Failed to get total lag for consumer group: {}", groupId, e); + return 0L; + } + } + + @SneakyThrows + public Map getConsumerGroupOffsets(String groupId) { + return getClient().listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata().get(requestTimeoutMs, TimeUnit.MILLISECONDS); + } + + /** + * Sync offsets from a fat group to a single-partition group + * Migration back from single-partition consumer to a fat group is not supported + * TODO: The best possible approach to synchronize the offsets is to do the synchronization as a part of the save Queue parameters with stop all consumers + * */ + public void syncOffsets(String fatGroupId, String newGroupId, Integer partitionId) { + try { + log.info("syncOffsets [{}][{}][{}]", fatGroupId, newGroupId, partitionId); + if (partitionId == null) { + return; + } + syncOffsetsUnsafe(fatGroupId, newGroupId, "." + partitionId); + } catch (Exception e) { + log.warn("Failed to syncOffsets from {} to {} partitionId {}", fatGroupId, newGroupId, partitionId, e); + } + } + + public void syncOffsetsUnsafe(String fatGroupId, String newGroupId, String topicSuffix) throws ExecutionException, InterruptedException, TimeoutException { + Map oldOffsets = getConsumerGroupOffsets(fatGroupId); + if (oldOffsets.isEmpty()) { + return; + } + + for (var consumerOffset : oldOffsets.entrySet()) { + var tp = consumerOffset.getKey(); + if (!tp.topic().endsWith(topicSuffix)) { + continue; + } + var om = consumerOffset.getValue(); + Map newOffsets = getConsumerGroupOffsets(newGroupId); + + var existingOffset = newOffsets.get(tp); + if (existingOffset == null) { + log.info("[{}] topic offset does not exists in the new node group {}, all found offsets {}", tp, newGroupId, newOffsets); + } else if (existingOffset.offset() >= om.offset()) { + log.info("[{}] topic offset {} >= than old node group offset {}", tp, existingOffset.offset(), om.offset()); + break; + } else { + log.info("[{}] SHOULD alter topic offset [{}] less than old node group offset [{}]", tp, existingOffset.offset(), om.offset()); + } + getClient().alterConsumerGroupOffsets(newGroupId, Map.of(tp, om)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); + log.info("[{}] altered new consumer groupId {}", tp, newGroupId); + break; + } + } + + public boolean isTopicEmpty(String topic) { + return areAllTopicsEmpty(Set.of(topic)); + } + + public boolean areAllTopicsEmpty(Set topics) { + try { + List existingTopics = getTopics().stream().filter(topics::contains).toList(); + if (existingTopics.isEmpty()) { + return true; + } + + List allPartitions = getClient().describeTopics(existingTopics).allTopicNames().get(requestTimeoutMs, TimeUnit.MILLISECONDS) + .entrySet().stream() + .flatMap(entry -> { + String topic = entry.getKey(); + TopicDescription topicDescription = entry.getValue(); + return topicDescription.partitions().stream().map(partitionInfo -> new TopicPartition(topic, partitionInfo.partition())); + }) + .toList(); + + Map beginningOffsets = getClient().listOffsets(allPartitions.stream() + .collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.earliest()))).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); + Map endOffsets = getClient().listOffsets(allPartitions.stream() + .collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.latest()))).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); + + for (TopicPartition partition : allPartitions) { + long beginningOffset = beginningOffsets.get(partition).offset(); + long endOffset = endOffsets.get(partition).offset(); + + if (beginningOffset != endOffset) { + log.debug("Partition [{}] of topic [{}] is not empty. Returning false.", partition.partition(), partition.topic()); + return false; + } + } + return true; + } catch (Exception e) { + log.error("Failed to check if topics [{}] empty.", topics, e); + return false; + } + } + + public void deleteConsumerGroup(String consumerGroupId) { + try { + getClient().deleteConsumerGroups(List.of(consumerGroupId)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); + } catch (Exception e) { + log.warn("Failed to delete consumer group {}", consumerGroupId, e); + } + } + + public AdminClient getClient() { + try { + return adminClient.get(); + } catch (ConcurrentException e) { + throw new RuntimeException("Failed to initialize Kafka admin client", e); + } + } + + @PreDestroy + private void destroy() throws Exception { + if (adminClient.isInitialized()) { + adminClient.get().close(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index 1e0064a5c8..65e9d5e4c4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -16,29 +16,12 @@ package org.thingsboard.server.queue.kafka; import lombok.Getter; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.ListOffsetsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.admin.OffsetSpec; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.TopicExistsException; import org.thingsboard.server.queue.TbEdgeQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.util.PropertyUtils; -import java.util.Collections; -import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. @@ -50,214 +33,42 @@ public class TbKafkaAdmin implements TbQueueAdmin, TbEdgeQueueAdmin { private final Map topicConfigs; @Getter private final int numPartitions; - private volatile Set topics; - - private final short replicationFactor; public TbKafkaAdmin(TbKafkaSettings settings, Map topicConfigs) { this.settings = settings; this.topicConfigs = topicConfigs; - String numPartitionsStr = topicConfigs.get(TbKafkaTopicConfigs.NUM_PARTITIONS_SETTING); if (numPartitionsStr != null) { numPartitions = Integer.parseInt(numPartitionsStr); } else { numPartitions = 1; } - replicationFactor = settings.getReplicationFactor(); } @Override - public void createTopicIfNotExists(String topic, String properties) { - Set topics = getTopics(); - if (topics.contains(topic)) { - return; - } - try { - Map configs = PropertyUtils.getProps(topicConfigs, properties); - configs.remove(TbKafkaTopicConfigs.NUM_PARTITIONS_SETTING); - NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(configs); - createTopic(newTopic).values().get(topic).get(); - topics.add(topic); - } catch (ExecutionException ee) { - if (ee.getCause() instanceof TopicExistsException) { - //do nothing - } else { - log.warn("[{}] Failed to create topic", topic, ee); - throw new RuntimeException(ee); - } - } catch (Exception e) { - log.warn("[{}] Failed to create topic", topic, e); - throw new RuntimeException(e); - } + public void createTopicIfNotExists(String topic, String properties, boolean force) { + settings.getAdmin().createTopicIfNotExists(topic, PropertyUtils.getProps(topicConfigs, properties), force); } @Override public void deleteTopic(String topic) { - Set topics = getTopics(); - if (topics.remove(topic)) { - settings.getAdminClient().deleteTopics(Collections.singletonList(topic)); - } else { - try { - if (settings.getAdminClient().listTopics().names().get().contains(topic)) { - settings.getAdminClient().deleteTopics(Collections.singletonList(topic)); - } else { - log.warn("Kafka topic [{}] does not exist.", topic); - } - } catch (InterruptedException | ExecutionException e) { - log.error("Failed to delete kafka topic [{}].", topic, e); - } - } - } - - private Set getTopics() { - if (topics == null) { - synchronized (this) { - if (topics == null) { - topics = ConcurrentHashMap.newKeySet(); - try { - topics.addAll(settings.getAdminClient().listTopics().names().get()); - } catch (InterruptedException | ExecutionException e) { - log.error("Failed to get all topics.", e); - } - } - } - } - return topics; - } - - public Set getAllTopics() { - try { - return settings.getAdminClient().listTopics().names().get(); - } catch (InterruptedException | ExecutionException e) { - log.error("Failed to get all topics.", e); - } - return null; - } - - public CreateTopicsResult createTopic(NewTopic topic) { - return settings.getAdminClient().createTopics(Collections.singletonList(topic)); + settings.getAdmin().deleteTopic(topic); } @Override public void destroy() { } - /** - * Sync offsets from a fat group to a single-partition group - * Migration back from single-partition consumer to a fat group is not supported - * TODO: The best possible approach to synchronize the offsets is to do the synchronization as a part of the save Queue parameters with stop all consumers - * */ - public void syncOffsets(String fatGroupId, String newGroupId, Integer partitionId) { - try { - log.info("syncOffsets [{}][{}][{}]", fatGroupId, newGroupId, partitionId); - if (partitionId == null) { - return; - } - syncOffsetsUnsafe(fatGroupId, newGroupId, "." + partitionId); - } catch (Exception e) { - log.warn("Failed to syncOffsets from {} to {} partitionId {}", fatGroupId, newGroupId, partitionId, e); - } - } - /** * Sync edge notifications offsets from a fat group to a single group per edge * */ public void syncEdgeNotificationsOffsets(String fatGroupId, String newGroupId) { try { log.info("syncEdgeNotificationsOffsets [{}][{}]", fatGroupId, newGroupId); - syncOffsetsUnsafe(fatGroupId, newGroupId, newGroupId); + settings.getAdmin().syncOffsetsUnsafe(fatGroupId, newGroupId, newGroupId); } catch (Exception e) { log.warn("Failed to syncEdgeNotificationsOffsets from {} to {}", fatGroupId, newGroupId, e); } } - @Override - public void deleteConsumerGroup(String consumerGroupId) { - try { - settings.getAdminClient().deleteConsumerGroups(Collections.singletonList(consumerGroupId)); - } catch (Exception e) { - log.warn("Failed to delete consumer group {}", consumerGroupId, e); - } - } - - void syncOffsetsUnsafe(String fatGroupId, String newGroupId, String topicSuffix) throws ExecutionException, InterruptedException, TimeoutException { - Map oldOffsets = getConsumerGroupOffsets(fatGroupId); - if (oldOffsets.isEmpty()) { - return; - } - - for (var consumerOffset : oldOffsets.entrySet()) { - var tp = consumerOffset.getKey(); - if (!tp.topic().endsWith(topicSuffix)) { - continue; - } - var om = consumerOffset.getValue(); - Map newOffsets = getConsumerGroupOffsets(newGroupId); - - var existingOffset = newOffsets.get(tp); - if (existingOffset == null) { - log.info("[{}] topic offset does not exists in the new node group {}, all found offsets {}", tp, newGroupId, newOffsets); - } else if (existingOffset.offset() >= om.offset()) { - log.info("[{}] topic offset {} >= than old node group offset {}", tp, existingOffset.offset(), om.offset()); - break; - } else { - log.info("[{}] SHOULD alter topic offset [{}] less than old node group offset [{}]", tp, existingOffset.offset(), om.offset()); - } - settings.getAdminClient().alterConsumerGroupOffsets(newGroupId, Map.of(tp, om)).all().get(10, TimeUnit.SECONDS); - log.info("[{}] altered new consumer groupId {}", tp, newGroupId); - break; - } - } - - @SneakyThrows - public Map getConsumerGroupOffsets(String groupId) { - return settings.getAdminClient().listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata().get(10, TimeUnit.SECONDS); - } - - public boolean isTopicEmpty(String topic) { - return areAllTopicsEmpty(Set.of(topic)); - } - - public boolean areAllTopicsEmpty(Set topics) { - try { - List existingTopics = getTopics().stream().filter(topics::contains).toList(); - if (existingTopics.isEmpty()) { - return true; - } - - List allPartitions = settings.getAdminClient().describeTopics(existingTopics).topicNameValues().entrySet().stream() - .flatMap(entry -> { - String topic = entry.getKey(); - TopicDescription topicDescription; - try { - topicDescription = entry.getValue().get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - return topicDescription.partitions().stream().map(partitionInfo -> new TopicPartition(topic, partitionInfo.partition())); - }) - .toList(); - - Map beginningOffsets = settings.getAdminClient().listOffsets(allPartitions.stream() - .collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.earliest()))).all().get(); - Map endOffsets = settings.getAdminClient().listOffsets(allPartitions.stream() - .collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.latest()))).all().get(); - - for (TopicPartition partition : allPartitions) { - long beginningOffset = beginningOffsets.get(partition).offset(); - long endOffset = endOffsets.get(partition).offset(); - - if (beginningOffset != endOffset) { - log.debug("Partition [{}] of topic [{}] is not empty. Returning false.", partition.partition(), partition.topic()); - return false; - } - } - return true; - } catch (InterruptedException | ExecutionException e) { - log.error("Failed to check if topics [{}] empty.", topics, e); - return false; - } - } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatisticConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatisticConfig.java index d44f9ee700..12a3ea8fa6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatisticConfig.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatisticConfig.java @@ -19,11 +19,11 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.util.TbKafkaComponent; @Component -@ConditionalOnProperty(prefix = "queue", value = "type", havingValue = "kafka") +@TbKafkaComponent @Getter @AllArgsConstructor @NoArgsConstructor diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java index 7a9c01b72f..e1cfb995c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java @@ -26,10 +26,10 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.queue.util.TbKafkaComponent; import java.time.Duration; import java.util.ArrayList; @@ -44,11 +44,12 @@ import java.util.concurrent.TimeUnit; @Slf4j @Component @RequiredArgsConstructor -@ConditionalOnProperty(prefix = "queue", value = "type", havingValue = "kafka") +@TbKafkaComponent public class TbKafkaConsumerStatsService { private final Set monitoredGroups = ConcurrentHashMap.newKeySet(); private final TbKafkaSettings kafkaSettings; + private final KafkaAdmin kafkaAdmin; private final TbKafkaConsumerStatisticConfig statsConfig; private Consumer consumer; @@ -77,7 +78,7 @@ public class TbKafkaConsumerStatsService { } for (String groupId : monitoredGroups) { try { - Map groupOffsets = kafkaSettings.getAdminClient().listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata() + Map groupOffsets = kafkaSettings.getAdmin().getClient().listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata() .get(statsConfig.getKafkaResponseTimeoutMs(), TimeUnit.MILLISECONDS); Map endOffsets = consumer.endOffsets(groupOffsets.keySet(), timeoutDuration); @@ -159,12 +160,14 @@ public class TbKafkaConsumerStatsService { @Override public String toString() { return "[" + - "topic=[" + topic + ']' + - ", partition=[" + partition + "]" + - ", committedOffset=[" + committedOffset + "]" + - ", endOffset=[" + endOffset + "]" + - ", lag=[" + lag + "]" + - "]"; + "topic=[" + topic + ']' + + ", partition=[" + partition + "]" + + ", committedOffset=[" + committedOffset + "]" + + ", endOffset=[" + endOffset + "]" + + ", lag=[" + lag + "]" + + "]"; } + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 06ccfe3a69..11736f68cf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -16,12 +16,10 @@ package org.thingsboard.server.queue.kafka; import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; @@ -30,12 +28,13 @@ import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.serialization.ByteArraySerializer; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.TbProperty; import org.thingsboard.server.queue.util.PropertyUtils; +import org.thingsboard.server.queue.util.TbKafkaComponent; import java.util.HashMap; import java.util.LinkedHashMap; @@ -47,7 +46,7 @@ import java.util.Properties; * Created by ashvayka on 25.09.18. */ @Slf4j -@ConditionalOnProperty(prefix = "queue", value = "type", havingValue = "kafka") +@TbKafkaComponent @ConfigurationProperties(prefix = "queue.kafka") @Component public class TbKafkaSettings { @@ -143,6 +142,9 @@ public class TbKafkaSettings { @Value("${queue.kafka.consumer-properties-per-topic-inline:}") private String consumerPropertiesPerTopicInline; + @Autowired + private KafkaAdmin kafkaAdmin; + @Deprecated @Setter private List other; @@ -150,8 +152,6 @@ public class TbKafkaSettings { @Setter private Map> consumerPropertiesPerTopic = new HashMap<>(); - private volatile AdminClient adminClient; - @PostConstruct public void initInlineTopicProperties() { Map> inlineProps = parseTopicPropertyList(consumerPropertiesPerTopicInline); @@ -240,15 +240,12 @@ public class TbKafkaSettings { } } - public AdminClient getAdminClient() { - if (adminClient == null) { - synchronized (this) { - if (adminClient == null) { - adminClient = AdminClient.create(toAdminProps()); - } - } - } - return adminClient; + /* + * Temporary solution to avoid major code changes. + * FIXME: use single instance of Kafka queue admin, don't create a separate one for each consumer/producer + * */ + public KafkaAdmin getAdmin() { + return kafkaAdmin; } protected Properties toAdminProps() { @@ -279,11 +276,4 @@ public class TbKafkaSettings { return result; } - @PreDestroy - private void destroy() { - if (adminClient != null) { - adminClient.close(); - } - } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 5d5834d20a..c50fd0d720 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -18,14 +18,14 @@ package org.thingsboard.server.queue.kafka; import jakarta.annotation.PostConstruct; import lombok.Getter; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import org.thingsboard.server.queue.util.PropertyUtils; +import org.thingsboard.server.queue.util.TbKafkaComponent; import java.util.Map; @Component -@ConditionalOnProperty(prefix = "queue", value = "type", havingValue = "kafka") +@TbKafkaComponent public class TbKafkaTopicConfigs { public static final String NUM_PARTITIONS_SETTING = "partitions"; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 75d53d9830..2c932534b1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -78,7 +78,7 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory templateBuilder.queueAdmin(new TbQueueAdmin() { @Override - public void createTopicIfNotExists(String topic, String properties) {} + public void createTopicIfNotExists(String topic, String properties, boolean force) {} @Override public void destroy() {} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 866f8d235e..245330cc3e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -58,6 +58,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.kafka.KafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -83,6 +84,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TopicService topicService; private final TbKafkaSettings kafkaSettings; + private final KafkaAdmin kafkaAdmin; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; @@ -118,7 +120,9 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final AtomicLong consumerCount = new AtomicLong(); private final AtomicLong edgeConsumerCount = new AtomicLong(); - public KafkaMonolithQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, + public KafkaMonolithQueueFactory(TopicService topicService, + TbKafkaSettings kafkaSettings, + KafkaAdmin kafkaAdmin, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -134,6 +138,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TasksQueueConfig tasksQueueConfig) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; + this.kafkaAdmin = kafkaAdmin; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -240,7 +245,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi String queueName = configuration.getName(); String groupId = topicService.buildConsumerGroupId("re-", configuration.getTenantId(), queueName, partitionId); - ruleEngineAdmin.syncOffsets(topicService.buildConsumerGroupId("re-", configuration.getTenantId(), queueName, null), // the fat groupId + kafkaAdmin.syncOffsets(topicService.buildConsumerGroupId("re-", configuration.getTenantId(), queueName, null), // the fat groupId groupId, partitionId); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 3b67ea4f9f..dbf4ab1aba 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -52,6 +52,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.kafka.KafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -74,6 +75,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TopicService topicService; private final TbKafkaSettings kafkaSettings; + private final KafkaAdmin kafkaAdmin; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; @@ -99,6 +101,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, + KafkaAdmin kafkaAdmin, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -111,6 +114,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; + this.kafkaAdmin = kafkaAdmin; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -234,7 +238,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { String queueName = configuration.getName(); String groupId = topicService.buildConsumerGroupId("re-", configuration.getTenantId(), queueName, partitionId); - ruleEngineAdmin.syncOffsets(topicService.buildConsumerGroupId("re-", configuration.getTenantId(), queueName, null), // the fat groupId + kafkaAdmin.syncOffsets(topicService.buildConsumerGroupId("re-", configuration.getTenantId(), queueName, null), // the fat groupId groupId, partitionId); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaComponent.java new file mode 100644 index 0000000000..ad4862e36d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaComponent.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Inherited +@Retention(RetentionPolicy.RUNTIME) +@Target({java.lang.annotation.ElementType.TYPE, java.lang.annotation.ElementType.METHOD}) +@ConditionalOnProperty(prefix = "queue", value = "type", havingValue = "kafka") +public @interface TbKafkaComponent {} diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java index ad026c63aa..bc37982245 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java @@ -28,7 +28,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.spy; -@SpringBootTest(classes = TbKafkaSettings.class) +@SpringBootTest(classes = {TbKafkaSettings.class, KafkaAdmin.class}) @TestPropertySource(properties = { "queue.type=kafka", "queue.kafka.bootstrap.servers=localhost:9092", diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/SslCredentialsWebServerCustomizer.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/SslCredentialsWebServerCustomizer.java index 6c73bb03de..fc64be6e8b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/SslCredentialsWebServerCustomizer.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/SslCredentialsWebServerCustomizer.java @@ -59,6 +59,7 @@ public class SslCredentialsWebServerCustomizer implements WebServerFactoryCustom public void customize(ConfigurableServletWebServerFactory factory) { SslCredentials sslCredentials = this.httpServerSslCredentialsConfig.getCredentials(); Ssl ssl = serverProperties.getSsl(); + ssl.setBundle("default"); ssl.setKeyAlias(sslCredentials.getKeyAlias()); ssl.setKeyPassword(sslCredentials.getKeyPassword()); factory.setSsl(ssl); diff --git a/common/util/src/main/java/org/thingsboard/common/util/CachedValue.java b/common/util/src/main/java/org/thingsboard/common/util/CachedValue.java new file mode 100644 index 0000000000..b0a41c2a42 --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/CachedValue.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.common.util; + +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; + +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +public class CachedValue { + + private final LoadingCache cache; + + public CachedValue(Supplier supplier, long valueTtlMs) { + this.cache = Caffeine.newBuilder() + .expireAfterWrite(valueTtlMs, TimeUnit.MILLISECONDS) + .build(__ -> supplier.get()); + } + + public V get() { + return cache.get(this); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 9803670d4b..62b35caa7f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -26,6 +26,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edqs.AttributeKv; @@ -118,7 +119,8 @@ public class BaseAttributesService implements AttributesService { List> futures = new ArrayList<>(attributes.size()); for (AttributeKvEntry attribute : attributes) { ListenableFuture future = Futures.transform(attributesDao.save(tenantId, entityId, scope, attribute), version -> { - edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attribute, version)); + TenantId edqsTenantId = entityId.getEntityType() == EntityType.TENANT ? (TenantId) entityId : tenantId; + edqsService.onUpdate(edqsTenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attribute, version)); return version; }, MoreExecutors.directExecutor()); futures.add(future); @@ -136,7 +138,8 @@ public class BaseAttributesService implements AttributesService { String key = keyVersionPair.getFirst(); Long version = keyVersionPair.getSecond(); if (version != null) { - edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + TenantId edqsTenantId = entityId.getEntityType() == EntityType.TENANT ? (TenantId) entityId : tenantId; + edqsService.onDelete(edqsTenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); } keys.add(key); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index d99413f13a..44b2daaf8e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -30,6 +30,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbCacheValueWrapper; import org.thingsboard.server.cache.VersionedTbCache; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edqs.AttributeKv; @@ -239,7 +240,8 @@ public class CachedAttributesService implements AttributesService { ListenableFuture future = Futures.transform(attributesDao.save(tenantId, entityId, scope, attribute), version -> { BaseAttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(((BaseAttributeKvEntry) attribute).getKv(), attribute.getLastUpdateTs(), version); put(entityId, scope, attributeKvEntry); - edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attributeKvEntry, version)); + TenantId edqsTenantId = entityId.getEntityType() == EntityType.TENANT ? (TenantId) entityId : tenantId; + edqsService.onUpdate(edqsTenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attributeKvEntry, version)); return version; }, cacheExecutor); futures.add(future); @@ -263,7 +265,8 @@ public class CachedAttributesService implements AttributesService { Long version = keyVersionPair.getSecond(); cache.evict(new AttributeCacheKey(scope, entityId, key), version); if (version != null) { - edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + TenantId edqsTenantId = entityId.getEntityType() == EntityType.TENANT ? (TenantId) entityId : tenantId; + edqsService.onDelete(edqsTenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); } return key; }, cacheExecutor)).toList()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java b/dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java index 90939858bc..ecf1376a73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java @@ -28,8 +28,11 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.dao.edge.stats.EdgeStatsCounterService; +import org.thingsboard.server.dao.edge.stats.EdgeStatsKey; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -41,6 +44,7 @@ public class PostgresEdgeEventService extends BaseEdgeEventService { private final EdgeEventDao edgeEventDao; private final ApplicationEventPublisher eventPublisher; + private final Optional statsCounterService; private ExecutorService edgeEventExecutor; @@ -64,6 +68,7 @@ public class PostgresEdgeEventService extends BaseEdgeEventService { Futures.addCallback(saveFuture, new FutureCallback<>() { @Override public void onSuccess(Void result) { + statsCounterService.ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1)); eventPublisher.publishEvent(SaveEntityEvent.builder() .tenantId(edgeEvent.getTenantId()) .entityId(edgeEvent.getEdgeId()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/stats/EdgeStatsCounterService.java b/dao/src/main/java/org/thingsboard/server/dao/edge/stats/EdgeStatsCounterService.java new file mode 100644 index 0000000000..16111cf514 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/stats/EdgeStatsCounterService.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.edge.stats; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.concurrent.ConcurrentHashMap; + +@ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true", matchIfMissing = false) +@Service +@Slf4j +@Getter +public class EdgeStatsCounterService { + + private final ConcurrentHashMap counterByEdge = new ConcurrentHashMap<>(); + + public void recordEvent(EdgeStatsKey type, TenantId tenantId, EdgeId edgeId, long value) { + MsgCounters counters = getOrCreateCounters(tenantId, edgeId); + switch (type) { + case DOWNLINK_MSGS_ADDED -> counters.getMsgsAdded().addAndGet(value); + case DOWNLINK_MSGS_PUSHED -> counters.getMsgsPushed().addAndGet(value); + case DOWNLINK_MSGS_PERMANENTLY_FAILED -> counters.getMsgsPermanentlyFailed().addAndGet(value); + case DOWNLINK_MSGS_TMP_FAILED -> counters.getMsgsTmpFailed().addAndGet(value); + } + } + + public void setDownlinkMsgsLag(TenantId tenantId, EdgeId edgeId, long value) { + getOrCreateCounters(tenantId, edgeId).getMsgsLag().set(value); + } + + public void clear(EdgeId edgeId) { + counterByEdge.remove(edgeId); + } + + public MsgCounters getOrCreateCounters(TenantId tenantId, EdgeId edgeId) { + return counterByEdge.computeIfAbsent(edgeId, id -> new MsgCounters(tenantId)); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/stats/EdgeStatsKey.java b/dao/src/main/java/org/thingsboard/server/dao/edge/stats/EdgeStatsKey.java new file mode 100644 index 0000000000..e017f6969e --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/stats/EdgeStatsKey.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.edge.stats; + +import lombok.Getter; + +@Getter +public enum EdgeStatsKey { + DOWNLINK_MSGS_ADDED("downlinkMsgsAdded"), + DOWNLINK_MSGS_PUSHED("downlinkMsgsPushed"), + DOWNLINK_MSGS_PERMANENTLY_FAILED("downlinkMsgsPermanentlyFailed"), + DOWNLINK_MSGS_TMP_FAILED("downlinkMsgsTmpFailed"), + DOWNLINK_MSGS_LAG("downlinkMsgsLag"); + + private final String key; + + EdgeStatsKey(String key) { + this.key = key; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/stats/MsgCounters.java b/dao/src/main/java/org/thingsboard/server/dao/edge/stats/MsgCounters.java new file mode 100644 index 0000000000..7596da0ecf --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/stats/MsgCounters.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.edge.stats; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.concurrent.atomic.AtomicLong; + +@Data +public class MsgCounters { + + private final TenantId tenantId; + private final AtomicLong msgsAdded = new AtomicLong(); + private final AtomicLong msgsPushed = new AtomicLong(); + private final AtomicLong msgsPermanentlyFailed = new AtomicLong(); + private final AtomicLong msgsTmpFailed = new AtomicLong(); + private final AtomicLong msgsLag = new AtomicLong(); + + public void clear() { + msgsAdded.set(0); + msgsPushed.set(0); + msgsPermanentlyFailed.set(0); + msgsTmpFailed.set(0); + msgsLag.set(0); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java index 1fbe682831..ff71be4298 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java @@ -26,9 +26,13 @@ import org.owasp.validator.html.ScanException; import org.thingsboard.server.common.data.validation.NoXss; import java.util.Optional; +import java.util.regex.Pattern; @Slf4j public class NoXssValidator implements ConstraintValidator { + + private static final Pattern JS_TEMPLATE_PATTERN = Pattern.compile("\\{\\{.*}}", Pattern.DOTALL); + private static final AntiSamy xssChecker = new AntiSamy(); private static final Policy xssPolicy; @@ -59,6 +63,9 @@ public class NoXssValidator implements ConstraintValidator { if (stringValue.isEmpty()) { return true; } + if (JS_TEMPLATE_PATTERN.matcher(stringValue).find()) { + return false; + } try { return xssChecker.scan(stringValue, xssPolicy).getNumberOfErrors() == 0; } catch (ScanException | PolicyException e) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index cecf4ab587..ceb7fcf822 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -196,7 +196,8 @@ public class BaseTimeseriesService implements TimeseriesService { if (saveLatest) { latestFutures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), version -> { if (version != null) { - edqsService.onUpdate(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, version)); + TenantId edqsTenantId = entityId.getEntityType() == EntityType.TENANT ? (TenantId) entityId : tenantId; + edqsService.onUpdate(edqsTenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, version)); } return version; }, MoreExecutors.directExecutor())); @@ -276,7 +277,8 @@ public class BaseTimeseriesService implements TimeseriesService { return Futures.transform(timeseriesLatestDao.removeLatest(tenantId, entityId, query), result -> { if (result.isRemoved()) { Long version = result.getVersion(); - edqsService.onDelete(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, query.getKey(), version)); + TenantId edqsTenantId = entityId.getEntityType() == EntityType.TENANT ? (TenantId) entityId : tenantId; + edqsService.onDelete(edqsTenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, query.getKey(), version)); } return result; }, MoreExecutors.directExecutor()); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index bbbd48aa49..32767043d7 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -486,6 +486,17 @@ public class DeviceServiceTest extends AbstractServiceTest { }); } + @Test + public void testSaveDeviceWithJSInjection_thenDataValidationException() { + Device device = new Device(); + device.setType("default"); + device.setTenantId(tenantId); + device.setName("{{constructor.constructor('location.href=\"https://evil.com\"')()}}"); + Assertions.assertThrows(DataValidationException.class, () -> { + deviceService.saveDevice(device); + }); + } + @Test public void testSaveDeviceWithInvalidTenant() { Device device = new Device(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java index 4877938d89..34da80e2db 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java @@ -35,7 +35,14 @@ public class NoXssValidatorTest { "

Link!!!

1221", "

Please log in to proceed

Username:

Password:



", " ", - "123 bebe" + "123 bebe", + "{{constructor.constructor('location.href=\"https://evil.com\"')()}}", + " {{constructor.constructor('alert(1)')()}}", + "{{}}", + "{{{constructor.constructor('location.href=\"https://evil.com\"')()}}}", + "test {{constructor.constructor('location.href=\"https://evil.com\"')()}} test", + "{{#if user}}Hello, {{user.name}}{{/if}}", + "{{ user.name }}" }) public void givenEntityWithMaliciousPropertyValue_thenReturnValidationError(String maliciousString) { Asset invalidAsset = new Asset(); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/notification/SlackService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/notification/SlackService.java index e68310fce9..1ac800f488 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/notification/SlackService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/notification/SlackService.java @@ -18,6 +18,7 @@ package org.thingsboard.rule.engine.api.notification; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.notification.targets.slack.SlackConversation; import org.thingsboard.server.common.data.notification.targets.slack.SlackConversationType; +import org.thingsboard.server.common.data.notification.targets.slack.SlackFile; import java.util.List; @@ -25,6 +26,8 @@ public interface SlackService { void sendMessage(TenantId tenantId, String token, String conversationId, String message); + void sendMessage(TenantId tenantId, String token, String conversationId, String message, List files); + List listConversations(TenantId tenantId, String token, SlackConversationType conversationType); String getToken(TenantId tenantId); diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c54bf038f2..f40a09c753 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -114,9 +114,9 @@ redis: # Minumum number of idle connections that can be maintained in the pool without being closed minIdle: "${REDIS_POOL_CONFIG_MIN_IDLE:16}" # Enable/Disable PING command send when a connection is borrowed - testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:true}" + testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:false}" # The property is used to specify whether to test the connection before returning it to the connection pool. - testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:true}" + testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:false}" # The property is used in the context of connection pooling in Redis testWhileIdle: "${REDIS_POOL_CONFIG_TEST_WHILE_IDLE:true}" # Minimum amount of time that an idle connection should be idle before it can be evicted from the connection pool. Value set in milliseconds diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 3a3725edef..587894d5ce 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -147,9 +147,9 @@ redis: # Minumum number of idle connections that can be maintained in the pool without being closed minIdle: "${REDIS_POOL_CONFIG_MIN_IDLE:16}" # Enable/Disable PING command send when a connection is borrowed - testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:true}" + testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:false}" # The property is used to specify whether to test the connection before returning it to the connection pool. - testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:true}" + testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:false}" # The property is used in the context of connection pooling in Redis testWhileIdle: "${REDIS_POOL_CONFIG_TEST_WHILE_IDLE:true}" # Minimum amount of time that an idle connection should be idle before it can be evicted from the connection pool. Value set in milliseconds diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 60568a6a4e..0895bfa676 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -114,9 +114,9 @@ redis: # Minumum number of idle connections that can be maintained in the pool without being closed minIdle: "${REDIS_POOL_CONFIG_MIN_IDLE:16}" # Enable/Disable PING command send when a connection is borrowed - testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:true}" + testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:false}" # The property is used to specify whether to test the connection before returning it to the connection pool. - testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:true}" + testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:false}" # The property is used in the context of connection pooling in Redis testWhileIdle: "${REDIS_POOL_CONFIG_TEST_WHILE_IDLE:true}" # Minimum amount of time that an idle connection should be idle before it can be evicted from the connection pool. Value set in milliseconds diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 55781abeeb..fae10cc892 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -115,9 +115,9 @@ redis: # Minumum number of idle connections that can be maintained in the pool without being closed minIdle: "${REDIS_POOL_CONFIG_MIN_IDLE:16}" # Enable/Disable PING command send when a connection is borrowed - testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:true}" + testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:false}" # The property is used to specify whether to test the connection before returning it to the connection pool. - testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:true}" + testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:false}" # The property is used in the context of connection pooling in Redis testWhileIdle: "${REDIS_POOL_CONFIG_TEST_WHILE_IDLE:true}" # Minimum amount of time that an idle connection should be idle before it can be evicted from the connection pool. Value set in milliseconds diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 746e8f2173..79aee31921 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -114,9 +114,9 @@ redis: # Minumum number of idle connections that can be maintained in the pool without being closed minIdle: "${REDIS_POOL_CONFIG_MIN_IDLE:16}" # Enable/Disable PING command send when a connection is borrowed - testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:true}" + testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:false}" # The property is used to specify whether to test the connection before returning it to the connection pool. - testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:true}" + testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:false}" # The property is used in the context of connection pooling in Redis testWhileIdle: "${REDIS_POOL_CONFIG_TEST_WHILE_IDLE:true}" # Minimum amount of time that an idle connection should be idle before it can be evicted from the connection pool. Value set in milliseconds diff --git a/ui-ngx/src/app/core/http/public-api.ts b/ui-ngx/src/app/core/http/public-api.ts index 63cc393ce6..43354f6b60 100644 --- a/ui-ngx/src/app/core/http/public-api.ts +++ b/ui-ngx/src/app/core/http/public-api.ts @@ -21,11 +21,13 @@ export * from './asset.service'; export * from './asset-profile.service'; export * from './attribute.service'; export * from './audit-log.service'; +export * from './calculated-fields.service'; export * from './component-descriptor.service'; export * from './customer.service'; export * from './dashboard.service'; export * from './device.service'; export * from './device-profile.service'; +export * from './domain.service'; export * from './entities-version-control.service'; export * from './entity.service'; export * from './edge.service'; @@ -34,6 +36,8 @@ export * from './entity-view.service'; export * from './event.service'; export * from './http-utils'; export * from './image.service'; +export * from './mobile-app.service'; +export * from './mobile-application.service'; export * from './notification.service'; export * from './oauth2.service'; export * from './ota-package.service'; @@ -42,6 +46,7 @@ export * from './resource.service'; export * from './rule-chain.service'; export * from './tenant.service'; export * from './tenant-profile.service'; +export * from './two-factor-authentication.service'; export * from './ui-settings.service'; export * from './user.service'; export * from './user-settings.service'; diff --git a/ui-ngx/src/app/modules/home/components/alarm/alarm-filter-config.component.scss b/ui-ngx/src/app/modules/home/components/alarm/alarm-filter-config.component.scss index 2ee3c0089d..c9e5eee944 100644 --- a/ui-ngx/src/app/modules/home/components/alarm/alarm-filter-config.component.scss +++ b/ui-ngx/src/app/modules/home/components/alarm/alarm-filter-config.component.scss @@ -13,6 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +@import '../scss/constants'; + :host { display: flex; max-width: 100%; @@ -39,16 +42,12 @@ tb-entity-subtype-list { flex: 1; - width: 180px; + @media #{$mat-gt-xs} { + width: 180px; + } .mdc-evolution-chip-set__chips { width: 100%; } } - - .mat-mdc-chip { - .mdc-evolution-chip__cell, .mat-mdc-chip-action, .mat-mdc-chip-action-label { - overflow: hidden; - } - } } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index b4b5a939e1..4086b3e7b0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -80,7 +80,7 @@ {{ 'entity.key' | translate }} - +
{{ argument.refEntityKey.key }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index ae8fd25170..430958d0f4 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -55,12 +55,6 @@ } :host ::ng-deep { - .mat-mdc-standard-chip { - .mdc-evolution-chip__cell--primary, .mdc-evolution-chip__action--primary, .mdc-evolution-chip__text-label { - overflow: hidden; - } - } - .arguments-table:not(.arguments-table-with-error) { .mdc-data-table__row:last-child .mat-mdc-cell { border-bottom: none; diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts index 60d05d4c86..a839e98c13 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts @@ -58,7 +58,7 @@ import { } from '@app/shared/models/dashboard.models'; import { WINDOW } from '@core/services/window.service'; import { WindowMessage } from '@shared/models/window-message.model'; -import { deepClone, guid, isDefined, isDefinedAndNotNull, isEqual, isNotEmptyStr } from '@app/core/utils'; +import { deepClone, guid, isDefined, isDefinedAndNotNull, isNotEmptyStr } from '@app/core/utils'; import { DashboardContext, DashboardPageInitData, @@ -119,7 +119,8 @@ import { } from '@home/components/dashboard-page/dashboard-settings-dialog.component'; import { ManageDashboardStatesDialogComponent, - ManageDashboardStatesDialogData + ManageDashboardStatesDialogData, + ManageDashboardStatesDialogResult } from '@home/components/dashboard-page/states/manage-dashboard-states-dialog.component'; import { ImportExportService } from '@shared/import-export/import-export.service'; import { AuthState } from '@app/core/auth/auth.models'; @@ -964,17 +965,17 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC $event.stopPropagation(); } this.dialog.open(ManageDashboardStatesDialogComponent, { + ManageDashboardStatesDialogResult>(ManageDashboardStatesDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { states: deepClone(this.dashboard.configuration.states), - widgets: deepClone(this.dashboard.configuration.widgets) as {[id: string]: Widget} + widgets: this.dashboard.configuration.widgets as {[id: string]: Widget} } }).afterClosed().subscribe((result) => { if (result) { - if (!isEqual(result.widgets, this.dashboard.configuration.widgets)) { - this.dashboard.configuration.widgets = result.widgets; + if (result.addWidgets) { + Object.assign(this.dashboard.configuration.widgets, result.addWidgets); } if (result.states) { this.updateStates(result.states); diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.html index a2b0a4c0f3..b91a62c8e7 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.html @@ -15,143 +15,134 @@ limitations under the License. --> - - -

dashboard.manage-states

- - -
- - -
-
-
-
-
- -
- dashboard.states - - - -
-
- -
- - -   - - - -
-
-
- - - {{ 'dashboard.state-name' | translate }} - - {{ state.name }} - - - - {{ 'dashboard.state-id' | translate }} - - {{ state.id }} - - - - {{ 'dashboard.is-root-state' | translate }} - - {{state.root ? 'check_box' : 'check_box_outline_blank'}} - - - - - - -
- - - -
-
-
- - -
- {{ 'dashboard.no-states-text' }} -
- - -
-
+ +

dashboard.manage-states

+ + +
+
+
+ +
+ dashboard.states + + +
-
-
-
- - + + +
+ + +   + + + +
+
+
+ + + {{ 'dashboard.state-name' | translate }} + + {{ state.name }} + + + + {{ 'dashboard.state-id' | translate }} + + {{ state.id }} + + + + {{ 'dashboard.is-root-state' | translate }} + + {{state.root ? 'check_box' : 'check_box_outline_blank'}} + + + + + + +
+ + + +
+
+
+ + +
+ {{ 'dashboard.no-states-text' }} +
+ +
- + +
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.scss b/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.scss index 862737c22b..98d48861bd 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.scss @@ -13,27 +13,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +@import "../scss/constants"; + :host { + height: 100%; + display: grid; + grid-template-rows: min-content auto min-content; + .manage-dashboard-states { - .tb-entity-table { - .tb-entity-table-content { - width: 100%; - height: 100%; - background: #fff; + .tb-entity-table-content { + width: 100%; + height: 100%; + background: #fff; - .tb-entity-table-title { - padding-right: 20px; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - } + .tb-entity-table-title { + padding-right: 20px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } - .table-container { - overflow: auto; - } + .table-container { + overflow: auto; } } } + + @media #{$mat-sm} { + min-width: 470px; + } + + @media #{$mat-gt-sm} { + min-width: 750px; + } } :host ::ng-deep { diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.ts b/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.ts index 7dddc18dbd..6e74ee9b14 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/states/manage-dashboard-states-dialog.component.ts @@ -14,21 +14,10 @@ /// limitations under the License. /// -import { - AfterViewInit, - Component, - ElementRef, - Inject, - OnInit, - SecurityContext, - SkipSelf, - ViewChild -} from '@angular/core'; -import { ErrorStateMatcher } from '@angular/material/core'; +import { AfterViewInit, Component, ElementRef, Inject, OnInit, SecurityContext, ViewChild } from '@angular/core'; import { MAT_DIALOG_DATA, MatDialog, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { FormGroupDirective, NgForm, UntypedFormBuilder, UntypedFormControl, UntypedFormGroup } from '@angular/forms'; import { Router } from '@angular/router'; import { DialogComponent } from '@app/shared/components/dialog.component'; import { DashboardState } from '@app/shared/models/dashboard.models'; @@ -44,7 +33,7 @@ import { fromEvent, merge } from 'rxjs'; import { debounceTime, distinctUntilChanged, tap } from 'rxjs/operators'; import { TranslateService } from '@ngx-translate/core'; import { DialogService } from '@core/services/dialog.service'; -import { deepClone, isDefined } from '@core/utils'; +import { deepClone, isDefined, isEqual } from '@core/utils'; import { DashboardStateDialogComponent, DashboardStateDialogData @@ -58,42 +47,42 @@ export interface ManageDashboardStatesDialogData { widgets: {[id: string]: Widget }; } +export interface ManageDashboardStatesDialogResult { + states: {[id: string]: DashboardState }; + addWidgets?: {[id: string]: Widget }; +} + @Component({ selector: 'tb-manage-dashboard-states-dialog', templateUrl: './manage-dashboard-states-dialog.component.html', - providers: [{provide: ErrorStateMatcher, useExisting: ManageDashboardStatesDialogComponent}], styleUrls: ['./manage-dashboard-states-dialog.component.scss'] }) export class ManageDashboardStatesDialogComponent - extends DialogComponent - implements OnInit, ErrorStateMatcher, AfterViewInit { + extends DialogComponent + implements OnInit, AfterViewInit { - statesFormGroup: UntypedFormGroup; + @ViewChild('searchInput', {static: false}) searchInputField: ElementRef; - states: {[id: string]: DashboardState }; - widgets: {[id: string]: Widget}; + @ViewChild(MatPaginator, {static: false}) paginator: MatPaginator; + @ViewChild(MatSort, {static: false}) sort: MatSort; + + isDirty = false; displayedColumns: string[]; pageLink: PageLink; textSearchMode = false; dataSource: DashboardStatesDatasource; - submitted = false; + private states: {[id: string]: DashboardState }; + private widgets: {[id: string]: Widget}; - stateNames: Set = new Set(); - - @ViewChild('searchInput') searchInputField: ElementRef; - - @ViewChild(MatPaginator) paginator: MatPaginator; - @ViewChild(MatSort) sort: MatSort; + private stateNames: Set = new Set(); + private addWidgets: {[id: string]: Widget} = {}; constructor(protected store: Store, protected router: Router, @Inject(MAT_DIALOG_DATA) public data: ManageDashboardStatesDialogData, - @SkipSelf() private errorStateMatcher: ErrorStateMatcher, - public dialogRef: MatDialogRef, - private fb: UntypedFormBuilder, + public dialogRef: MatDialogRef, private translate: TranslateService, private dialogs: DialogService, private utils: UtilsService, @@ -103,7 +92,6 @@ export class ManageDashboardStatesDialogComponent this.states = this.data.states; this.widgets = this.data.widgets; - this.statesFormGroup = this.fb.group({}); Object.values(this.states).forEach(value => this.stateNames.add(value.name)); const sortOrder: SortOrder = { property: 'name', direction: Direction.ASC }; @@ -258,77 +246,65 @@ export class ManageDashboardStatesDialogComponent } duplicateState($event: Event, state: DashboardStateInfo) { - const originalState = state; - const newStateName = this.getNextDuplicatedName(state.name); - if (newStateName) { - const newStateId = newStateName.toLowerCase().replace(/\W/g, '_'); - if (this.states[newStateId]) { - this.stateNames.add(newStateName); - this.duplicateState(null, state); + const suffix = ` - ${this.translate.instant('action.copy')} `; + let counter = 0; + const maxAttempts = 1000; + + while (counter++ < maxAttempts) { + const candidateName = `${state.name}${suffix}${counter}`; + if (this.stateNames.has(candidateName)) continue; + + const candidateId = candidateName.toLowerCase().replace(/\W/g, '_'); + if (this.states[candidateId]) { + continue; } - const duplicatedStates = deepClone(originalState); - const duplicatedWidgets = deepClone(this.widgets); + + const duplicatedState = deepClone(state); const mainWidgets = {}; const rightWidgets = {}; - duplicatedStates.id = newStateId; - duplicatedStates.name = newStateName; - duplicatedStates.root = false; - this.stateNames.add(duplicatedStates.name); + duplicatedState.id = candidateId; + duplicatedState.name = candidateName; + duplicatedState.root = false; + this.stateNames.add(duplicatedState.name); - for (const [key, value] of Object.entries(duplicatedStates.layouts.main.widgets)) { + for (const [key, value] of Object.entries(duplicatedState.layouts.main.widgets)) { const guid = this.utils.guid(); mainWidgets[guid] = value; - duplicatedWidgets[guid] = this.widgets[key]; - duplicatedWidgets[guid].id = guid; + this.addWidgets[guid] = deepClone(this.widgets[key] ?? this.addWidgets[key]); + this.addWidgets[guid].id = guid; } - duplicatedStates.layouts.main.widgets = mainWidgets; + duplicatedState.layouts.main.widgets = mainWidgets; - if (isDefined(duplicatedStates.layouts?.right)) { - for (const [key, value] of Object.entries(duplicatedStates.layouts.right.widgets)) { + if (isDefined(duplicatedState.layouts?.right)) { + for (const [key, value] of Object.entries(duplicatedState.layouts.right.widgets)) { const guid = this.utils.guid(); rightWidgets[guid] = value; - duplicatedWidgets[guid] = this.widgets[key]; - duplicatedWidgets[guid].id = guid; + this.addWidgets[guid] = deepClone(this.widgets[key] ?? this.addWidgets[key]); + this.addWidgets[guid].id = guid; } - duplicatedStates.layouts.right.widgets = rightWidgets; + duplicatedState.layouts.right.widgets = rightWidgets; } - this.states[duplicatedStates.id] = duplicatedStates; - this.widgets = duplicatedWidgets; + this.states[duplicatedState.id] = duplicatedState; this.onStatesUpdated(); + return; } } - private getNextDuplicatedName(stateName: string): string { - const suffix = ` - ${this.translate.instant('action.copy')} `; - let counter = 0; - while (++counter < Number.MAX_SAFE_INTEGER) { - const newName = `${stateName}${suffix}${counter}`; - if (!this.stateNames.has(newName)) { - return newName; - } - } - - return null; - } - private onStatesUpdated() { - this.statesFormGroup.markAsDirty(); + this.isDirty = true; this.updateData(true); } - isErrorState(control: UntypedFormControl | null, form: FormGroupDirective | NgForm | null): boolean { - const originalErrorState = this.errorStateMatcher.isErrorState(control, form); - const customErrorState = !!(control && control.invalid && this.submitted); - return originalErrorState || customErrorState; - } - cancel(): void { this.dialogRef.close(null); } save(): void { - this.submitted = true; - this.dialogRef.close({ states: this.states, widgets: this.widgets }); + const result: ManageDashboardStatesDialogResult = {states: this.states}; + if (!isEqual(this.addWidgets, {})) { + result.addWidgets = this.addWidgets; + } + this.dialogRef.close(result); } } diff --git a/ui-ngx/src/app/modules/home/components/entity/entity-filter-view.component.ts b/ui-ngx/src/app/modules/home/components/entity/entity-filter-view.component.ts index ac7333079d..7eae04452d 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entity-filter-view.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entity-filter-view.component.ts @@ -128,6 +128,9 @@ export class EntityFilterViewComponent implements ControlValueAccessor { {edgeTypes}); } break; + case AliasFilterType.apiUsageState: + this.filterDisplayValue = this.translate.instant('alias.filter-type-apiUsageState'); + break; case AliasFilterType.entityViewType: const entityViewTypesQuoted = []; this.filter.entityViewTypes.forEach((entityViewType) => { diff --git a/ui-ngx/src/app/modules/home/components/relation/relation-filters.component.ts b/ui-ngx/src/app/modules/home/components/relation/relation-filters.component.ts index 92cd675ee8..e2d19b9e6c 100644 --- a/ui-ngx/src/app/modules/home/components/relation/relation-filters.component.ts +++ b/ui-ngx/src/app/modules/home/components/relation/relation-filters.component.ts @@ -128,7 +128,7 @@ export class RelationFiltersComponent extends PageComponent implements ControlVa entityTypes: [filter ? filter.entityTypes : []] }); if (this.enableNotOption) { - formGroup.addControl('negate', this.fb.control({value: filter ? filter.negate : false, disabled: true})); + formGroup.addControl('negate', this.fb.control({value: filter?.negate ?? false, disabled: !filter?.relationType})); formGroup.get('relationType').valueChanges.pipe( takeUntil(this.destroy$) ).subscribe(value => { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/alias/entity-alias-select.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/alias/entity-alias-select.component.ts index 1af25bd01e..d65d4067e5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/alias/entity-alias-select.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/alias/entity-alias-select.component.ts @@ -251,7 +251,6 @@ export class EntityAliasSelectComponent implements ControlValueAccessor, OnInit, }, 0); } } else { - this.entityAliasList.push(newAlias); this.modelValue = newAlias.id; this.selectEntityAliasFormGroup.get('entityAlias').patchValue(newAlias, {emitEvent: true}); this.propagateChange(this.modelValue); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.html index 5dbeb8b646..5307a18a7b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.html @@ -116,7 +116,7 @@
dynamic-form.property.disable-on-property
- + {{ 'dynamic-form.property.disable-on-property-none' | translate }} {{ prop }} diff --git a/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts b/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts index 98595de798..0f13409a3e 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts @@ -40,6 +40,7 @@ export interface TemplateNotificationDialogData { predefinedType?: NotificationType; isAdd?: boolean; isCopy?: boolean; + name?: string; } @Component({ @@ -85,6 +86,9 @@ export class TemplateNotificationDialogComponent this.hideSelectType = true; this.templateNotificationForm.get('notificationType').setValue(this.data.predefinedType, {emitEvent: false}); } + if (isDefinedAndNotNull(this.data?.name)) { + this.templateNotificationForm.get('name').setValue(this.data.name, {emitEvent: false}); + } if (data.isAdd || data.isCopy) { this.dialogTitle = 'notification.add-notification-template'; diff --git a/ui-ngx/src/app/modules/home/pages/widget/widget-type-autocomplete.component.scss b/ui-ngx/src/app/modules/home/pages/widget/widget-type-autocomplete.component.scss index e28d8ec14a..c8f678d679 100644 --- a/ui-ngx/src/app/modules/home/pages/widget/widget-type-autocomplete.component.scss +++ b/ui-ngx/src/app/modules/home/pages/widget/widget-type-autocomplete.component.scss @@ -24,7 +24,8 @@ } .tb-widget-type-option-image-preview { width: 36px; - max-height: 100%; + height: 100%; + max-height: 36px; object-fit: contain; border-radius: 6px; } diff --git a/ui-ngx/src/app/shared/components/entity/entity-list.component.html b/ui-ngx/src/app/shared/components/entity/entity-list.component.html index 9510f2a952..6bf7cdb78d 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-list.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-list.component.html @@ -25,6 +25,7 @@ {{entity.name}} diff --git a/ui-ngx/src/app/shared/components/entity/entity-subtype-list.component.html b/ui-ngx/src/app/shared/components/entity/entity-subtype-list.component.html index 595010fb19..6635921d7f 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-subtype-list.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-subtype-list.component.html @@ -22,6 +22,7 @@ {{customTranslate(entitySubtype)}} diff --git a/ui-ngx/src/app/shared/components/notification/template-autocomplete.component.html b/ui-ngx/src/app/shared/components/notification/template-autocomplete.component.html index d5e67a1f60..d712f11114 100644 --- a/ui-ngx/src/app/shared/components/notification/template-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/notification/template-autocomplete.component.html @@ -29,7 +29,7 @@ (click)="clear()"> close -