From 1b9dfa17c25c188d67a1b86076c765da5538bed9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 9 Oct 2023 18:24:43 +0300 Subject: [PATCH 01/33] Refactoring for latencies tracking --- .../monitoring/data/Latencies.java | 4 ++ .../thingsboard/monitoring/data/Latency.java | 48 ++----------------- .../notification/HighLatencyNotification.java | 10 ++-- .../monitoring/service/BaseHealthChecker.java | 2 +- .../service/BaseMonitoringService.java | 5 +- .../service/MonitoringReporter.java | 26 +++++----- 6 files changed, 30 insertions(+), 65 deletions(-) diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/Latencies.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/Latencies.java index 3370d42462..52e73a64cd 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/Latencies.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/Latencies.java @@ -25,4 +25,8 @@ public class Latencies { return String.format("%sRequest", key); } + public static String wsUpdate(String key) { + return String.format("%sWsUpdate", key); + } + } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/Latency.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/Latency.java index c64291e30f..4b17bf179f 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/Latency.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/Latency.java @@ -15,53 +15,15 @@ */ package org.thingsboard.monitoring.data; -import com.google.common.util.concurrent.AtomicDouble; -import lombok.RequiredArgsConstructor; +import lombok.Data; -import java.util.concurrent.atomic.AtomicInteger; - -@RequiredArgsConstructor +@Data(staticConstructor = "of") public class Latency { - private final String key; - private final AtomicDouble latencySum = new AtomicDouble(); - private final AtomicInteger counter = new AtomicInteger(); - - public synchronized void report(double latencyInMs) { - latencySum.addAndGet(latencyInMs); - counter.incrementAndGet(); - } - - public synchronized double getAvg() { - return latencySum.get() / counter.get(); - } - - public boolean isNotEmpty() { - return counter.get() > 0; - } - - public synchronized void reset() { - latencySum.set(0.0); - counter.set(0); - } - - public String getKey() { - return key; - } - - public synchronized Latency snapshot() { - Latency snapshot = new Latency(key); - snapshot.latencySum.set(latencySum.get()); - snapshot.counter.set(counter.get()); - return snapshot; - } + private final double value; - @Override - public String toString() { - return "Latency{" + - "key='" + key + '\'' + - ", avgLatency=" + getAvg() + - '}'; + public String getFormattedValue() { + return String.format("%,.2f ms", value); } } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/HighLatencyNotification.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/HighLatencyNotification.java index 939cb7440f..5744e40d41 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/HighLatencyNotification.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/HighLatencyNotification.java @@ -21,11 +21,11 @@ import java.util.Collection; public class HighLatencyNotification implements Notification { - private final Collection latencies; + private final Collection highLatencies; private final int thresholdMs; - public HighLatencyNotification(Collection latencies, int thresholdMs) { - this.latencies = latencies; + public HighLatencyNotification(Collection highLatencies, int thresholdMs) { + this.highLatencies = highLatencies; this.thresholdMs = thresholdMs; } @@ -33,8 +33,8 @@ public class HighLatencyNotification implements Notification { public String getText() { StringBuilder text = new StringBuilder(); text.append("Some of the latencies are higher than ").append(thresholdMs).append(" ms:\n"); - latencies.forEach(latency -> { - text.append(String.format("[%s] %,.2f ms\n", latency.getKey(), latency.getAvg())); + highLatencies.forEach(latency -> { + text.append(String.format("[%s] %s\n", latency.getKey(), latency.getFormattedValue())); }); return text.toString(); } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java index affca1ce09..697573289d 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java @@ -96,7 +96,7 @@ public abstract class BaseHealthChecker, T extends MonitoringTarget> { - @Autowired + @Autowired(required = false) private List configs; private final List> healthCheckers = new LinkedList<>(); private final List devices = new LinkedList<>(); @@ -54,6 +54,9 @@ public abstract class BaseMonitoringService, T ext @PostConstruct private void init() { + if (configs == null || configs.isEmpty()) { + return; + } tbClient.logIn(); configs.forEach(config -> { config.getTargets().forEach(target -> { diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java index f41454a76a..aabf04caee 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java @@ -63,25 +63,20 @@ public class MonitoringReporter { private String reportingAssetId; public void reportLatencies(TbClient tbClient) { - List latencies = this.latencies.values().stream() - .filter(Latency::isNotEmpty) - .map(latency -> { - Latency snapshot = latency.snapshot(); - latency.reset(); - return snapshot; - }) - .collect(Collectors.toList()); if (latencies.isEmpty()) { return; } - log.info("Latencies:\n{}", latencies.stream().map(latency -> latency.getKey() + ": " + latency.getAvg() + " ms") + log.debug("Latencies:\n{}", latencies.values().stream().map(latency -> latency.getKey() + ": " + latency.getFormattedValue()) .collect(Collectors.joining("\n")) + "\n"); - if (!latencyReportingEnabled) return; - if (latencies.stream().anyMatch(latency -> latency.getAvg() >= (double) latencyThresholdMs)) { - HighLatencyNotification highLatencyNotification = new HighLatencyNotification(latencies, latencyThresholdMs); + List highLatencies = latencies.values().stream() + .filter(latency -> latency.getValue() >= (double) latencyThresholdMs) + .collect(Collectors.toList()); + if (!highLatencies.isEmpty()) { + HighLatencyNotification highLatencyNotification = new HighLatencyNotification(highLatencies, latencyThresholdMs); notificationService.sendNotification(highLatencyNotification); + log.warn("{}", highLatencyNotification.getText()); } try { @@ -99,10 +94,11 @@ public class MonitoringReporter { } ObjectNode msg = JacksonUtil.newObjectNode(); - latencies.forEach(latency -> { - msg.set(latency.getKey(), new DoubleNode(latency.getAvg())); + latencies.values().forEach(latency -> { + msg.set(latency.getKey(), new DoubleNode(latency.getValue())); }); tbClient.saveEntityTelemetry(new AssetId(UUID.fromString(reportingAssetId)), "time", msg); + latencies.clear(); } catch (Exception e) { log.error("Failed to report latencies: {}", e.getMessage()); } @@ -112,7 +108,7 @@ public class MonitoringReporter { String latencyKey = key + "Latency"; double latencyInMs = (double) latencyInNanos / 1000_000; log.trace("Reporting latency [{}]: {} ms", key, latencyInMs); - latencies.computeIfAbsent(latencyKey, k -> new Latency(latencyKey)).report(latencyInMs); + latencies.put(latencyKey, Latency.of(latencyKey, latencyInMs)); } public void serviceFailure(Object serviceKey, Throwable error) { From 7559aa9fe37d32a62eca4a36ee2344d8a154cfe0 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Fri, 10 Nov 2023 14:45:48 +0200 Subject: [PATCH 02/33] Added gateways dashboard creation on tenant add --- .../data/json/demo/dashboards/gateways.json | 1315 ----------------- .../dashboards/gateways.json} | 26 +- .../tenant/DefaultTbTenantService.java | 1 + .../service/install/InstallScripts.java | 9 + 4 files changed, 23 insertions(+), 1328 deletions(-) delete mode 100644 application/src/main/data/json/demo/dashboards/gateways.json rename application/src/main/data/json/{demo/dashboards/gateway.json => tenant/dashboards/gateways.json} (99%) diff --git a/application/src/main/data/json/demo/dashboards/gateways.json b/application/src/main/data/json/demo/dashboards/gateways.json deleted file mode 100644 index f4eb275c3c..0000000000 --- a/application/src/main/data/json/demo/dashboards/gateways.json +++ /dev/null @@ -1,1315 +0,0 @@ -{ - "title": "Gateways", - "image": null, - "mobileHide": false, - "mobileOrder": null, - "configuration": { - "widgets": { - "94715984-ae74-76e4-20b7-2f956b01ed80": { - "type": "latest", - "sizeX": 24, - "sizeY": 12, - "config": { - "timewindow": { - "displayValue": "", - "selectedTab": 0, - "realtime": { - "realtimeType": 1, - "interval": 1000, - "timewindowMs": 86400000, - "quickInterval": "CURRENT_DAY" - }, - "history": { - "historyType": 0, - "interval": 1000, - "timewindowMs": 60000, - "fixedTimewindow": { - "startTimeMs": 1694085270425, - "endTimeMs": 1694171670425 - }, - "quickInterval": "CURRENT_DAY" - }, - "aggregation": { - "type": "NONE", - "limit": 200 - } - }, - "showTitle": true, - "backgroundColor": "rgb(255, 255, 255)", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "4px", - "settings": { - "enableSearch": true, - "displayPagination": true, - "defaultPageSize": 10, - "defaultSortOrder": "entityName", - "displayEntityName": true, - "displayEntityType": false, - "entitiesTitle": "List of gateways", - "enableSelectColumnDisplay": true, - "displayEntityLabel": false, - "entityNameColumnTitle": "Gateway Name" - }, - "title": "Devices gateway table", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400, - "padding": "5px 10px 5px 10px" - }, - "useDashboardTimewindow": false, - "showLegend": false, - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "active", - "type": "attribute", - "label": "Active", - "color": "#2196f3", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": true, - "useCellContentFunction": true, - "cellContentFunction": "value = '⬤';\nreturn value;", - "cellStyleFunction": "var color;\nif (value == 'false') {\n color = '#EB5757';\n} else {\n color = '#27AE60';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" - }, - "_hash": 0.3646047595211721 - }, - { - "name": "eventsSent", - "type": "timeseries", - "label": "Sent", - "color": "#4caf50", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.7235710720767985 - }, - { - "name": "eventsProduced", - "type": "timeseries", - "label": "Events", - "color": "#f44336", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.5085933386303254 - }, - { - "name": "LOGS", - "type": "timeseries", - "label": "Latest log", - "color": "#ffc107", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.3504240371585048, - "postFuncBody": "if(value) {\n return value.substring(0, 31) + \"...\";\n} else {\n return '';\n}" - }, - { - "name": "RemoteLoggingLevel", - "type": "attribute", - "label": "Log level", - "color": "#607d8b", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.9785994222542516 - } - ], - "entityAliasId": "3e0f533a-0db1-3292-184f-06e73535061a" - } - ], - "showTitleIcon": true, - "titleIcon": "list", - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "List device", - "widgetStyle": {}, - "displayTimewindow": true, - "actions": { - "headerButton": [ - { - "name": "Add device", - "icon": "add", - "type": "customPretty", - "customHtml": "
\n \n

Add device

\n \n \n \n \n \n
\n
\n
\n \n Device name\n \n \n Device name is required.\n \n \n
\n \n Latitude\n \n \n \n Longitude\n \n \n
\n \n Label\n \n \n
\n
\n
\n \n \n \n
\n
\n", - "customCss": "", - "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet customDialog = $injector.get(widgetContext.servicesMap.get('customDialog'));\nlet deviceService = $injector.get(widgetContext.servicesMap.get('deviceService'));\nlet attributeService = $injector.get(widgetContext.servicesMap.get('attributeService'));\n\nopenAddDeviceDialog();\n\nfunction openAddDeviceDialog() {\n customDialog.customDialog(htmlTemplate, AddDeviceDialogController).subscribe();\n}\n\nfunction AddDeviceDialogController(instance) {\n let vm = instance;\n \n vm.addDeviceFormGroup = vm.fb.group({\n deviceName: ['', [vm.validators.required]],\n deviceLabel: [''],\n attributes: vm.fb.group({\n latitude: [null],\n longitude: [null]\n }) \n });\n \n vm.cancel = function() {\n vm.dialogRef.close(null);\n };\n \n vm.save = function() {\n vm.addDeviceFormGroup.markAsPristine();\n let device = {\n additionalInfo: {gateway: true},\n name: vm.addDeviceFormGroup.get('deviceName').value,\n type: 'gateway',\n label: vm.addDeviceFormGroup.get('deviceLabel').value\n };\n deviceService.saveDevice(device).subscribe(\n function (device) {\n saveAttributes(device.id).subscribe(\n function () {\n widgetContext.updateAliases();\n vm.dialogRef.close(null);\n }\n );\n }\n );\n };\n \n function saveAttributes(entityId) {\n let attributes = vm.addDeviceFormGroup.get('attributes').value;\n let attributesArray = [];\n for (let key in attributes) {\n attributesArray.push({key: key, value: attributes[key]});\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId, \"SERVER_SCOPE\", attributesArray);\n } else {\n return widgetContext.rxjs.of([]);\n }\n }\n}\n", - "customResources": [], - "id": "70837a9d-c3de-a9a7-03c5-dccd14998758" - } - ], - "actionCellButton": [ - { - "id": "78845501-234e-a452-6819-82b5b776e99f", - "name": "Configuration", - "icon": "settings", - "type": "openDashboardState", - "targetDashboardStateId": "__entityname__config", - "openRightLayout": false, - "setEntityId": true - }, - { - "id": "f6ffdba8-e40f-2b8d-851b-f5ecaf18606b", - "name": "Graphs", - "icon": "show_chart", - "type": "openDashboardState", - "targetDashboardStateId": "__entityname_grafic", - "setEntityId": true - }, - { - "name": "Edit device", - "icon": "edit", - "type": "customPretty", - "customHtml": "
\n \n

Edit device

\n \n \n \n \n \n
\n
\n
\n \n Device name\n \n \n Device name is required.\n \n \n
\n \n Latitude\n \n \n \n Longitude\n \n \n
\n \n Label\n \n \n
\n
\n
\n \n \n \n
\n
\n", - "customCss": "/*=======================================================================*/\n/*========== There are two examples: for edit and add entity ==========*/\n/*=======================================================================*/\n/*======================== Edit entity example ========================*/\n/*=======================================================================*/\n/*\n.edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n\n.relations-list.old-relations tb-entity-select tb-entity-autocomplete button {\n display: none;\n} \n*/\n/*========================================================================*/\n/*========================= Add entity example =========================*/\n/*========================================================================*/\n/*\n.add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n*/\n", - "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet customDialog = $injector.get(widgetContext.servicesMap.get('customDialog'));\nlet deviceService = $injector.get(widgetContext.servicesMap.get('deviceService'));\nlet attributeService = $injector.get(widgetContext.servicesMap.get('attributeService'));\n\nopenEditDeviceDialog();\n\nfunction openEditDeviceDialog() {\n customDialog.customDialog(htmlTemplate, EditDeviceDialogController).subscribe();\n}\n\nfunction EditDeviceDialogController(instance) {\n let vm = instance;\n \n vm.device = null;\n vm.attributes = {};\n \n vm.editDeviceFormGroup = vm.fb.group({\n deviceName: ['', [vm.validators.required]],\n deviceLabel: [''],\n attributes: vm.fb.group({\n latitude: [null],\n longitude: [null]\n }) \n });\n \n vm.cancel = function() {\n vm.dialogRef.close(null);\n };\n \n vm.save = function() {\n vm.editDeviceFormGroup.markAsPristine();\n vm.device.name = vm.editDeviceFormGroup.get('deviceName').value;\n vm.device.label = vm.editDeviceFormGroup.get('deviceLabel').value;\n deviceService.saveDevice(vm.device).subscribe(\n function () {\n saveAttributes().subscribe(\n function () {\n widgetContext.updateAliases();\n vm.dialogRef.close(null);\n }\n );\n }\n );\n };\n \n getEntityInfo();\n \n function getEntityInfo() {\n deviceService.getDevice(entityId.id).subscribe(\n function (device) {\n attributeService.getEntityAttributes(entityId, 'SERVER_SCOPE',\n ['latitude', 'longitude']).subscribe(\n function (attributes) {\n for (let i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n vm.device = device;\n vm.editDeviceFormGroup.patchValue(\n {\n deviceName: vm.device.name,\n deviceLabel: vm.device.label,\n attributes: {\n latitude: vm.attributes.latitude,\n longitude: vm.attributes.longitude\n }\n }, {emitEvent: false}\n );\n } \n );\n }\n ); \n }\n \n function saveAttributes() {\n let attributes = vm.editDeviceFormGroup.get('attributes').value;\n let attributesArray = [];\n for (let key in attributes) {\n attributesArray.push({key: key, value: attributes[key]});\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId, 'SERVER_SCOPE', attributesArray);\n } else {\n return widgetContext.rxjs.of([]);\n }\n }\n}\n", - "customResources": [], - "id": "242671f3-76c6-6982-7acc-6f12addf0ccc" - }, - { - "name": "Delete device", - "icon": "delete", - "type": "custom", - "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet dialogs = $injector.get(widgetContext.servicesMap.get('dialogs'));\nlet deviceService = $injector.get(widgetContext.servicesMap.get('deviceService'));\n\nopenDeleteDeviceDialog();\n\nfunction openDeleteDeviceDialog() {\n let title = \"Are you sure you want to delete the device \" + entityName + \"?\";\n let content = \"Be careful, after the confirmation, the device and all related data will become unrecoverable!\";\n dialogs.confirm(title, content, 'Cancel', 'Delete').subscribe(\n function (result) {\n if (result) {\n deleteDevice();\n }\n }\n );\n}\n\nfunction deleteDevice() {\n deviceService.deleteDevice(entityId.id).subscribe(\n function () {\n widgetContext.updateAliases();\n }\n );\n}\n", - "id": "862ec2b7-fbcf-376e-f85f-b77c07f36efa" - } - ], - "rowClick": [ - { - "id": "ad5fc7e1-5e60-e056-6940-a75a383466a1", - "name": "to_entityname__config", - "icon": "more_horiz", - "type": "openDashboardState", - "targetDashboardStateId": "__entityname__config", - "setEntityId": true, - "stateEntityParamName": "" - } - ] - } - }, - "id": "94715984-ae74-76e4-20b7-2f956b01ed80", - "typeFullFqn": "system.entity_admin_widgets.device_admin_table" - }, - "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { - "type": "timeseries", - "sizeX": 14, - "sizeY": 13, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "LOGS", - "type": "timeseries", - "label": "LOGS", - "color": "#2196f3", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.3496649158709739, - "postFuncBody": "return value.replace(/ - (.*) - \\[/gi, ' - $1 - [');" - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 2592000000 - }, - "aggregation": { - "type": "NONE", - "limit": 200 - } - }, - "showTitle": true, - "backgroundColor": "rgb(255, 255, 255)", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "showTimestamp": true, - "displayPagination": true, - "defaultPageSize": 10 - }, - "title": "Debug events (logs)", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": false, - "showLegend": false, - "widgetStyle": {}, - "actions": {}, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true - }, - "id": "eadabbc7-519e-76fc-ba10-b3fe8c18da10", - "typeFullFqn": "system.cards.timeseries_table" - }, - "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { - "type": "timeseries", - "sizeX": 17, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "opcuaEventsProduced", - "type": "timeseries", - "label": "opcuaEventsProduced", - "color": "#2196f3", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.1477920581839779 - }, - { - "name": "opcuaEventsSent", - "type": "timeseries", - "label": "opcuaEventsSent", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.6500957113784758 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 120000 - }, - "aggregation": { - "type": "NONE", - "limit": 25000 - }, - "hideInterval": false, - "hideAggregation": false - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - }, - "showLegend": true, - "legendConfig": { - "direction": "column", - "position": "right", - "showMin": true, - "showMax": true, - "showAvg": true, - "showTotal": true - } - }, - "title": "Real time information", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": false, - "displayTimewindow": true, - "actions": {} - }, - "id": "f928afc4-30d1-8d0c-e3cf-777f9f9d1155", - "typeFullFqn": "system.charts.basic_timeseries" - }, - "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { - "type": "timeseries", - "sizeX": 7, - "sizeY": 7, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "eventsSent", - "type": "timeseries", - "label": "Events", - "color": "#2196f3", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.8156044798125357 - }, - { - "name": "eventsProduced", - "type": "timeseries", - "label": "Produced", - "color": "#4caf50", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.6538259344015449 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 604800000 - }, - "aggregation": { - "type": "NONE", - "limit": 200 - } - }, - "showTitle": true, - "backgroundColor": "rgb(255, 255, 255)", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "showTimestamp": true, - "displayPagination": true, - "defaultPageSize": 6, - "hideEmptyLines": true - }, - "title": "Total Messages", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": false, - "showLegend": false, - "widgetStyle": {}, - "actions": {}, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true, - "legendConfig": { - "direction": "column", - "position": "bottom", - "showMin": false, - "showMax": false, - "showAvg": true, - "showTotal": false - } - }, - "id": "2a95b473-042d-59d0-2da2-40d0cccb6c8a", - "typeFullFqn": "system.cards.timeseries_table" - }, - "aaa69366-aacc-9028-65aa-645c0f8533ec": { - "type": "timeseries", - "sizeX": 17, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "eventsSent", - "type": "timeseries", - "label": "eventsSent", - "color": "#2196f3", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.41414001784591314 - }, - { - "name": "eventsProduced", - "type": "timeseries", - "label": "eventsProduced", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.7819101846284422 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - }, - "showLegend": true, - "legendConfig": { - "direction": "column", - "position": "right", - "showMin": true, - "showMax": true, - "showAvg": true, - "showTotal": true - } - }, - "title": "History information", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": true, - "displayTimewindow": true, - "actions": {} - }, - "id": "aaa69366-aacc-9028-65aa-645c0f8533ec", - "typeFullFqn": "system.charts.basic_timeseries" - }, - "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { - "type": "timeseries", - "sizeX": 17, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "bleEventsProduced", - "type": "timeseries", - "label": "bleEventsProduced", - "color": "#2196f3", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.5625165504526104 - }, - { - "name": "bleEventsSent", - "type": "timeseries", - "label": "bleEventsSent", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.6817950080745288 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 5000, - "timewindowMs": 120000 - }, - "aggregation": { - "type": "AVG", - "limit": 25000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - }, - "showLegend": true, - "legendConfig": { - "direction": "column", - "position": "right", - "showMin": true, - "showMax": true, - "showAvg": true, - "showTotal": true - } - }, - "title": "Real time information", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": false, - "displayTimewindow": true, - "actions": {} - }, - "id": "ce5c7d01-a3ef-5cf0-4578-8505135c23a0", - "typeFullFqn": "system.charts.basic_timeseries" - }, - "466f046d-6005-a168-b107-60fcb2469cd5": { - "type": "latest", - "sizeX": 7, - "sizeY": 5, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "displayValue": "", - "selectedTab": 0, - "realtime": { - "realtimeType": 1, - "interval": 1000, - "timewindowMs": 60000, - "quickInterval": "CURRENT_DAY" - }, - "history": { - "historyType": 0, - "interval": 1000, - "timewindowMs": 60000, - "fixedTimewindow": { - "startTimeMs": 1694085270425, - "endTimeMs": 1694171670425 - }, - "quickInterval": "CURRENT_DAY" - }, - "aggregation": { - "type": "AVG", - "limit": 25000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "eventsTitle": "Gateway Events Form", - "eventsReg": [ - "EventsProduced", - "EventsSent" - ] - }, - "title": "Gateway events", - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "dropShadow": true, - "enableFullscreen": true, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "displayTimewindow": true, - "showLegend": false, - "actions": {} - }, - "id": "466f046d-6005-a168-b107-60fcb2469cd5", - "typeFullFqn": "system.gateway_widgets.attributes_card" - }, - "8fc32225-164f-3258-73f7-e6b6d959cf0b": { - "type": "latest", - "sizeX": 10, - "sizeY": 9, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "displayValue": "", - "selectedTab": 0, - "realtime": { - "realtimeType": 1, - "interval": 1000, - "timewindowMs": 60000, - "quickInterval": "CURRENT_DAY" - }, - "history": { - "historyType": 0, - "interval": 1000, - "timewindowMs": 60000, - "fixedTimewindow": { - "startTimeMs": 1694085270425, - "endTimeMs": 1694171670425 - }, - "quickInterval": "CURRENT_DAY" - }, - "aggregation": { - "type": "AVG", - "limit": 25000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "gatewayTitle": "Gateway configuration (Single device)", - "readOnly": false - }, - "title": "New Gateway configuration (Single device)", - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "dropShadow": true, - "enableFullscreen": true, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "displayTimewindow": true, - "showLegend": false, - "actions": {} - }, - "id": "8fc32225-164f-3258-73f7-e6b6d959cf0b", - "typeFullFqn": "system.gateway_widgets.config_form_latest" - }, - "063fc179-c9fd-f952-e714-f24e9c43c05c": { - "type": "rpc", - "sizeX": 4, - "sizeY": 2, - "config": { - "targetDeviceAliases": [], - "showTitle": false, - "backgroundColor": "#e6e7e8", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "0px", - "settings": { - "requestTimeout": 5000, - "oneWayElseTwoWay": true, - "styleButton": { - "isRaised": true, - "isPrimary": false - }, - "methodParams": "{}", - "methodName": "gateway_reboot", - "buttonText": "GATEWAY REBOOT" - }, - "title": "New RPC Button", - "dropShadow": true, - "enableFullscreen": false, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "showLegend": false, - "actions": {}, - "datasources": [], - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true, - "targetDeviceAliasIds": [ - "b2487e75-2fa4-f211-142c-434dfd50c70c" - ] - }, - "id": "063fc179-c9fd-f952-e714-f24e9c43c05c", - "typeFullFqn": "system.control_widgets.rpcbutton" - }, - "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { - "type": "rpc", - "sizeX": 4, - "sizeY": 2, - "config": { - "targetDeviceAliases": [], - "showTitle": false, - "backgroundColor": "#e6e7e8", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "0px", - "settings": { - "requestTimeout": 5000, - "oneWayElseTwoWay": true, - "styleButton": { - "isRaised": true, - "isPrimary": false - }, - "methodName": "gateway_restart", - "methodParams": "{}", - "buttonText": "GATEWAY RESTART" - }, - "title": "New RPC Button", - "dropShadow": true, - "enableFullscreen": false, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "showLegend": false, - "actions": {}, - "datasources": [], - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true, - "targetDeviceAliasIds": [ - "b2487e75-2fa4-f211-142c-434dfd50c70c" - ] - }, - "id": "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7", - "typeFullFqn": "system.control_widgets.rpcbutton" - }, - "6770b6ba-eff8-df05-75f8-c1f9326d4842": { - "type": "latest", - "sizeX": 6, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "latitude", - "type": "attribute", - "label": "latitude", - "color": "#2196f3", - "settings": {}, - "_hash": 0.9743324774725604 - }, - { - "name": "longitude", - "type": "attribute", - "label": "longitude", - "color": "#4caf50", - "settings": {}, - "_hash": 0.5530093635101525 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "displayValue": "", - "selectedTab": 0, - "realtime": { - "realtimeType": 1, - "interval": 1000, - "timewindowMs": 60000, - "quickInterval": "CURRENT_DAY" - }, - "history": { - "historyType": 0, - "interval": 1000, - "timewindowMs": 60000, - "fixedTimewindow": { - "startTimeMs": 1694085270425, - "endTimeMs": 1694171670425 - }, - "quickInterval": "CURRENT_DAY" - }, - "aggregation": { - "type": "AVG", - "limit": 25000 - } - }, - "showTitle": false, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "fitMapBounds": true, - "latKeyName": "latitude", - "lngKeyName": "longitude", - "showLabel": true, - "label": "${entityName}", - "tooltipPattern": "${entityName}

Latitude: ${latitude:7}
Longitude: ${longitude:7}

Delete", - "markerImageSize": 34, - "useColorFunction": false, - "markerImages": [], - "useMarkerImageFunction": false, - "color": "#fe7569", - "mapProvider": "OpenStreetMap.Mapnik", - "showTooltip": true, - "autocloseTooltip": true, - "defaultCenterPosition": [ - 0, - 0 - ], - "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", - "showTooltipAction": "click", - "polygonKeyName": "coordinates", - "polygonOpacity": 0.5, - "polygonStrokeOpacity": 1, - "polygonStrokeWeight": 1, - "zoomOnClick": true, - "showCoverageOnHover": true, - "animate": true, - "maxClusterRadius": 80, - "removeOutsideVisibleBounds": true, - "defaultZoomLevel": 5 - }, - "title": "Gateway Location", - "dropShadow": true, - "enableFullscreen": false, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "showLegend": false, - "widgetStyle": {}, - "actions": { - "tooltipAction": [ - { - "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", - "name": "delete", - "icon": "more_horiz", - "type": "custom", - "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" - } - ] - }, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true - }, - "id": "6770b6ba-eff8-df05-75f8-c1f9326d4842", - "typeFullFqn": "system.input_widgets.markers_placement_openstreetmap" - } - }, - "states": { - "main_gateway": { - "name": "Gateways", - "root": true, - "layouts": { - "main": { - "widgets": { - "94715984-ae74-76e4-20b7-2f956b01ed80": { - "sizeX": 24, - "sizeY": 12, - "row": 0, - "col": 0 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "backgroundSizeMode": "100%", - "autoFillHeight": true, - "mobileAutoFillHeight": false, - "mobileRowHeight": 70, - "margin": 10, - "outerMargin": true - } - } - } - }, - "__entityname__config": { - "name": "${entityName} Configuration", - "root": false, - "layouts": { - "main": { - "widgets": { - "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { - "sizeX": 14, - "sizeY": 13, - "row": 0, - "col": 10 - }, - "8fc32225-164f-3258-73f7-e6b6d959cf0b": { - "sizeX": 10, - "sizeY": 9, - "row": 0, - "col": 0 - }, - "063fc179-c9fd-f952-e714-f24e9c43c05c": { - "sizeX": 4, - "sizeY": 2, - "row": 9, - "col": 0 - }, - "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { - "sizeX": 4, - "sizeY": 2, - "row": 11, - "col": 0 - }, - "6770b6ba-eff8-df05-75f8-c1f9326d4842": { - "sizeX": 6, - "sizeY": 4, - "row": 9, - "col": 4 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "backgroundSizeMode": "100%", - "autoFillHeight": true, - "mobileAutoFillHeight": false, - "mobileRowHeight": 70, - "margin": 10, - "outerMargin": true - } - } - } - }, - "__entityname_grafic": { - "name": "${entityName} Details", - "root": false, - "layouts": { - "main": { - "widgets": { - "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { - "sizeX": 17, - "sizeY": 4, - "mobileHeight": null, - "row": 4, - "col": 7 - }, - "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { - "sizeX": 7, - "sizeY": 7, - "row": 5, - "col": 0 - }, - "aaa69366-aacc-9028-65aa-645c0f8533ec": { - "sizeX": 17, - "sizeY": 4, - "mobileHeight": null, - "row": 0, - "col": 7 - }, - "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { - "sizeX": 17, - "sizeY": 4, - "mobileHeight": null, - "row": 8, - "col": 7 - }, - "466f046d-6005-a168-b107-60fcb2469cd5": { - "sizeX": 7, - "sizeY": 5, - "row": 0, - "col": 0 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "backgroundSizeMode": "auto 100%", - "autoFillHeight": true, - "mobileAutoFillHeight": true, - "mobileRowHeight": 70, - "margin": 10, - "outerMargin": true - } - } - } - } - }, - "entityAliases": { - "3e0f533a-0db1-3292-184f-06e73535061a": { - "id": "3e0f533a-0db1-3292-184f-06e73535061a", - "alias": "Gateways", - "filter": { - "type": "deviceType", - "resolveMultiple": true, - "deviceNameFilter": "", - "deviceTypes": [ - "gateway" - ] - } - }, - "b2487e75-2fa4-f211-142c-434dfd50c70c": { - "id": "b2487e75-2fa4-f211-142c-434dfd50c70c", - "alias": "Current Gateway", - "filter": { - "type": "stateEntity", - "resolveMultiple": false, - "stateEntityParamName": "", - "defaultStateEntity": null - } - } - }, - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 86400000 - }, - "aggregation": { - "type": "NONE", - "limit": 25000 - }, - "hideInterval": false, - "hideAggregation": false, - "hideAggInterval": false - }, - "settings": { - "stateControllerId": "entity", - "showTitle": true, - "showDashboardsSelect": true, - "showEntitiesSelect": true, - "showDashboardTimewindow": true, - "showDashboardExport": true, - "toolbarAlwaysOpen": true, - "titleColor": "rgba(0,0,0,0.870588)" - }, - "filters": {} - }, - "externalId": null, - "name": "Gateways" -} \ No newline at end of file diff --git a/application/src/main/data/json/demo/dashboards/gateway.json b/application/src/main/data/json/tenant/dashboards/gateways.json similarity index 99% rename from application/src/main/data/json/demo/dashboards/gateway.json rename to application/src/main/data/json/tenant/dashboards/gateways.json index b35b480ef2..972aed299f 100644 --- a/application/src/main/data/json/demo/dashboards/gateway.json +++ b/application/src/main/data/json/tenant/dashboards/gateways.json @@ -1,5 +1,5 @@ { - "title": "Gateway", + "title": "ThingsBoard IoT Gateways", "image": null, "mobileHide": false, "mobileOrder": null, @@ -40,7 +40,7 @@ "color": "rgba(0, 0, 0, 0.87)", "padding": "4px", "settings": { - "entitiesTitle": "Gateway list", + "entitiesTitle": "Gateways list", "enableSearch": true, "enableSelectColumnDisplay": false, "enableStickyHeader": true, @@ -55,7 +55,7 @@ "defaultSortOrder": "entityName", "useRowStyleFunction": false }, - "title": "New Entities table", + "title": "Gateways list", "dropShadow": true, "enableFullscreen": false, "titleStyle": { @@ -571,11 +571,11 @@ "padding": "8px", "settings": { "useMarkdownTextFunction": true, - "markdownTextFunction": "var blockData = '';\nvar connectorsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action=>action.name==\"Connecotrs\");\nvar logsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action=>action.name==\"Logs\");\nfunction generateMatHeader(index) {\n if( index !== undefined && index > -1) {\n return ``\n } else {\n return \"\" \n }\n}\nfunction createDataBlock(value, label, dividerStyle, mobile, index) {\n blockData += `\n \n
\n \n ${generateMatHeader(index)}\n ${label}\n \n ${value}\n `;\n}\ncreateDataBlock(data[0].Status, \"Status\", data[0].Status === \"Active\"? 'divider-green' : 'divider-red');\ncreateDataBlock(data[0].Name, \"Gateway Name\", '', ctx.isMobile);\ncreateDataBlock(data[0].Type, \"Gateway Type\", '');\ncreateDataBlock(\n `${(data[1]?data[1].count:0)} `\n + \" | \" + \n `${(data[2]?data[2][\"count 2\"]:0)} `\n , \"Devices (Active | Inactive)\", '');\ncreateDataBlock(\n `${(data[0].active_connectors?JSON.parse(data[0].active_connectors).length:0)} `\n + \" | \" + \n `${(data[0].inactive_connectors?JSON.parse(data[0].inactive_connectors).length:0)} `\n , \"Connectors (Active | Inactive)\", '', '', connectorsIndex);\ncreateDataBlock(data[0].ALL_ERRORS_COUNT || 0, \"Errors\", (data[0].ALL_ERRORS_COUNT || 0) === 0 ? 'divider-green' : 'divider-red', '', logsIndex);\nreturn `
${blockData}
`;", + "markdownTextFunction": "var blockData = '';\nvar connectorsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action=>action.name==\"Connectors\");\nvar logsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action=>action.name==\"Logs\");\nfunction generateMatHeader(index) {\n if( index !== undefined && index > -1) {\n return ``\n } else {\n return \"\" \n }\n}\nfunction createDataBlock(value, label, dividerStyle, mobile, index) {\n blockData += `\n \n
\n \n ${generateMatHeader(index)}\n ${label}\n \n ${value}\n `;\n}\ncreateDataBlock(data[0].Status, \"Status\", data[0].Status === \"Active\"? 'divider-green' : 'divider-red');\ncreateDataBlock(data[0].Name, \"Gateway Name\", '', ctx.isMobile);\ncreateDataBlock(data[0].Type, \"Gateway Type\", '');\ncreateDataBlock(\n `${(data[1]?data[1].count:0)} `\n + \" | \" + \n `${(data[2]?data[2][\"count 2\"]:0)} `\n , \"Devices (Active | Inactive)\", '');\ncreateDataBlock(\n `${(data[0].active_connectors?JSON.parse(data[0].active_connectors).length:0)} `\n + \" | \" + \n `${(data[0].inactive_connectors?JSON.parse(data[0].inactive_connectors).length:0)} `\n , \"Connectors (Active | Inactive)\", '', '', connectorsIndex);\ncreateDataBlock(data[0].ALL_ERRORS_COUNT || 0, \"Errors\", (data[0].ALL_ERRORS_COUNT || 0) === 0 ? 'divider-green' : 'divider-red', '', logsIndex);\nreturn `
${blockData}
`;", "applyDefaultMarkdownStyle": false, "markdownCss": ".divider {\n position: absolute;\n width: 3px;\n top: 8px;\n border-radius: 2px;\n bottom: 8px;\n border: 1px solid rgba(31, 70, 144, 1);\n background-color: rgba(31, 70, 144, 1);\n left: 10px;\n}\n.divider-green .divider {\n border: 1px solid rgb(25,128,56);\n background-color: rgb(25,128,56);\n}\n\n.divider-green .mat-mdc-card-content {\n color: rgb(25,128,56);\n}\n\n.divider-red .divider {\n border: 1px solid rgb(203,37,48);\n background-color: rgb(203,37,48);\n}\n\n.divider-red .mat-mdc-card-content {\n color: rgb(203,37,48);\n}\n\n.mdc-card {\n position: relative;\n padding-left: 10px;\n margin-bottom: 1px;\n}\n\n.mat-mdc-card-subtitle {\n font-weight: 400;\n font-size: 12px;\n}\n\n.mat-mdc-card-header {\n padding: 8px 16px 0;\n}\n\n.mat-mdc-card-content:last-child {\n padding-bottom: 8px;\n font-size: 16px;\n}\n\n.cards-container {\n height: calc(100% - 1px);\n justify-content: stretch;\n align-items: center;\n margin-bottom: 1px;\n}\n\n::ng-deep.tb-home-widget-link > div {\n flex-grow: 1;\n cursor: pointer;\n}\n\n .tb-home-widget-link {\n width: 100%;\n }\n\n .tb-home-widget-link:hover::after{\n color: inherit;\n }\n \n .tb-home-widget-link::after{\n content: 'arrow_forward';\n display: inline-block;\n transform: rotate(315deg);\n font-family: 'Material Icons';\n font-weight: normal;\n font-style: normal;\n font-size: 18px;\n color: rgba(0, 0, 0, 0.12);\n vertical-align: bottom;\n margin-left: 6px;\n}" }, - "title": "New Markdown/HTML Card", + "title": "Connectors", "showTitleIcon": false, "iconColor": "rgba(0, 0, 0, 0.87)", "iconSize": "24px", @@ -599,7 +599,7 @@ "actions": { "elementClick": [ { - "name": "Connecotrs", + "name": "Connectors", "icon": "more_horiz", "useShowWidgetActionFunction": null, "showWidgetActionFunction": "return true;", @@ -680,7 +680,7 @@ "defaultSortOrder": "-createdTime", "useRowStyleFunction": false }, - "title": "New Alarms table", + "title": "Alarms", "dropShadow": true, "enableFullscreen": false, "titleStyle": { @@ -1074,7 +1074,7 @@ } ] }, - "title": "New RPC remote shell", + "title": "RPC remote shell", "dropShadow": true, "enableFullscreen": true, "widgetStyle": { @@ -1485,7 +1485,7 @@ } ] }, - "title": "New RPC debug terminal", + "title": "RPC debug terminal", "dropShadow": true, "enableFullscreen": true, "widgetStyle": {}, @@ -1868,7 +1868,7 @@ "applyDefaultMarkdownStyle": false, "markdownCss": ".action-buttons-container {\r\n display: flex;\r\n flex-wrap: wrap;\r\n flex-direction: row;\r\n height: 100%;\r\n width: 100%;\r\n align-content: center;\r\n}\r\n\r\nbutton {\r\n flex-grow: 1;\r\n margin: 10px;\r\n min-width: 150px;\r\n height: auto;\r\n}" }, - "title": "New Markdown/HTML Card", + "title": "Service command", "showTitleIcon": false, "iconColor": "rgba(0, 0, 0, 0.87)", "iconSize": "24px", @@ -1980,7 +1980,7 @@ "applyDefaultMarkdownStyle": false, "markdownCss": ".action-buttons-container {\r\n display: flex;\r\n flex-wrap: wrap;\r\n flex-direction: row;\r\n height: 100%;\r\n width: 100%;\r\n align-content: start;\r\n}\r\n\r\nbutton {\r\n flex-grow: 1;\r\n margin: 10px;\r\n min-width: 150px;\r\n height: auto;\r\n}" }, - "title": "New Markdown/HTML Card", + "title": "General configuration", "showTitleIcon": false, "iconColor": "rgba(0, 0, 0, 0.87)", "iconSize": "24px", @@ -2145,7 +2145,7 @@ "applyDefaultMarkdownStyle": true, "markdownCss": ".mat-mdc-form-field-subscript-wrapper {\n display: none !important;\n}" }, - "title": "New Markdown/HTML Card", + "title": "Gateway devices", "showTitleIcon": false, "iconColor": "rgba(0, 0, 0, 0.87)", "iconSize": "24px", @@ -4939,7 +4939,7 @@ "applyDefaultMarkdownStyle": false, "markdownCss": ".action-container {\r\n display: flex;\r\n flex-wrap: wrap;\r\n flex-direction: row;\r\n height: 100%;\r\n width: 100%;\r\n}\r\n\r\nbutton {\r\n flex-grow: 1;\r\n margin: 10px;\r\n min-width: 150px;\r\n height: auto;\r\n}" }, - "title": "New Markdown/HTML Card", + "title": "Gateway commands", "showTitleIcon": false, "iconColor": "rgba(0, 0, 0, 0.87)", "iconSize": "24px", diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java index 19d4126989..3267ed1c17 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java @@ -54,6 +54,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T if (created) { installScripts.createDefaultRuleChains(savedTenant.getId()); installScripts.createDefaultEdgeRuleChains(savedTenant.getId()); + installScripts.createDefaultTenantDashboards(savedTenant.getId(), null); } tenantProfileCache.evict(savedTenant.getId()); notificationEntityService.notifyCreateOrUpdateTenant(savedTenant, created ? diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 7f0b70cb27..96a6678789 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -249,6 +249,15 @@ public class InstallScripts { public void loadDashboards(TenantId tenantId, CustomerId customerId) throws Exception { Path dashboardsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, DASHBOARDS_DIR); + loadDashboardsFromDir(tenantId, customerId, dashboardsDir); + } + + public void createDefaultTenantDashboards(TenantId tenantId, CustomerId customerId) throws Exception { + Path dashboardsDir = Paths.get(getDataDir(), JSON_DIR, TENANT_DIR, DASHBOARDS_DIR); + loadDashboardsFromDir(tenantId, customerId, dashboardsDir); + } + + private void loadDashboardsFromDir(TenantId tenantId, CustomerId customerId, Path dashboardsDir) throws IOException { try (DirectoryStream dirStream = Files.newDirectoryStream(dashboardsDir, path -> path.toString().endsWith(JSON_EXT))) { dirStream.forEach( path -> { From 316a09854cb23918afe3610a941e86b00bfed547 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Sun, 12 Nov 2023 22:15:57 +0100 Subject: [PATCH 03/33] implemented device actor delete msg and fixed assign to tenant --- .../server/actors/device/DeviceActor.java | 3 ++ .../server/actors/tenant/TenantActor.java | 14 ++++++++ .../DefaultTbNotificationEntityService.java | 3 +- .../device/DefaultTbDeviceService.java | 2 ++ .../queue/DefaultTbClusterService.java | 15 +++++--- .../server/service/queue/ProtoUtils.java | 22 ++++++++++++ .../server/cluster/TbClusterService.java | 4 ++- common/cluster-api/src/main/proto/queue.proto | 8 +++++ .../server/common/msg/MsgType.java | 2 ++ .../msg/rule/engine/DeviceDeleteMsg.java | 36 +++++++++++++++++++ 10 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/rule/engine/DeviceDeleteMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java index 48425d58db..d69b9aec0b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java @@ -64,6 +64,9 @@ public class DeviceActor extends ContextAwareActor { case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: processor.processAttributesUpdate((DeviceAttributesEventNotificationMsg) msg); break; + case DEVICE_DELETE_TO_DEVICE_ACTOR_MSG: + ctx.stop(ctx.getSelf()); + break; case DEVICE_CREDENTIALS_UPDATE_TO_DEVICE_ACTOR_MSG: processor.processCredentialsUpdate(msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index c9f5448df3..f508d967c9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -53,10 +53,13 @@ import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; +import java.util.HashSet; import java.util.List; +import java.util.Set; @Slf4j public class TenantActor extends RuleChainManagerActor { @@ -65,8 +68,11 @@ public class TenantActor extends RuleChainManagerActor { private boolean isCore; private ApiUsageState apiUsageState; + private Set deletedDevices; + private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext, tenantId); + this.deletedDevices = new HashSet<>(); } boolean cantFindTenant = false; @@ -221,6 +227,10 @@ public class TenantActor extends RuleChainManagerActor { if (!isCore) { log.warn("RECEIVED INVALID MESSAGE: {}", msg); } + if (deletedDevices.contains(msg.getDeviceId())) { + log.warn("RECEIVED MESSAGE FOR DELETED DEVICE: {}", msg); + return; + } TbActorRef deviceActor = getOrCreateDeviceActor(msg.getDeviceId()); if (priority) { deviceActor.tellWithHighPriority(msg); @@ -249,6 +259,10 @@ public class TenantActor extends RuleChainManagerActor { Edge edge = systemContext.getEdgeService().findEdgeById(tenantId, edgeId); edgeRpcService.updateEdge(tenantId, edge); } + } else if (msg.getEntityId().getEntityType() == EntityType.DEVICE && ComponentLifecycleEvent.DELETED == msg.getEvent()) { + DeviceId deviceId = (DeviceId) msg.getEntityId(); + onToDeviceActorMsg(new DeviceDeleteMsg(tenantId, deviceId), true); + deletedDevices.add(deviceId); } else if (isRuleEngine) { TbActorRef target = getEntityActorRef(msg.getEntityId()); if (target != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java index 72e8f44020..8682d18a62 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java @@ -112,7 +112,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS public void notifyDeleteDevice(TenantId tenantId, DeviceId deviceId, CustomerId customerId, Device device, User user, Object... additionalInfo) { gatewayNotificationsService.onDeviceDeleted(device); - tbClusterService.onDeviceDeleted(device, null); + tbClusterService.onDeviceDeleted(tenantId, device, null); logEntityAction(tenantId, deviceId, device, customerId, ActionType.DELETED, user, additionalInfo); } @@ -126,6 +126,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS @Override public void notifyAssignDeviceToTenant(TenantId tenantId, TenantId newTenantId, DeviceId deviceId, CustomerId customerId, Device device, Tenant tenant, User user, Object... additionalInfo) { + tbClusterService.onDeviceAssignedToTenant(tenantId, device); logEntityAction(tenantId, deviceId, device, customerId, ActionType.ASSIGNED_TO_TENANT, user, additionalInfo); pushAssignedFromNotification(tenant, newTenantId, device); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java index ac2d656ba6..a3d04f6344 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java @@ -232,6 +232,8 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T Tenant tenant = tenantService.findTenantById(tenantId); Device assignedDevice = deviceService.assignDeviceToTenant(newTenantId, device); + + notificationEntityService.notifyAssignDeviceToTenant(tenantId, newTenantId, deviceId, assignedDevice.getCustomerId(), assignedDevice, tenant, user, newTenantId.toString(), newTenant.getName()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 103d2b68e2..75a1baa5c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -308,10 +308,17 @@ public class DefaultTbClusterService implements TbClusterService { } @Override - public void onDeviceDeleted(Device device, TbQueueCallback callback) { - broadcastEntityDeleteToTransport(device.getTenantId(), device.getId(), device.getName(), callback); - sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), false, false, true); - broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), ComponentLifecycleEvent.DELETED); + public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { + DeviceId deviceId = device.getId(); + broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); + sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); + broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); + } + + @Override + public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { + onDeviceDeleted(oldTenantId, device, null); + sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), true, false, false); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java index 76a8e3e89a..8986ca8b51 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg; import org.thingsboard.server.common.msg.rpc.RemoveRpcActorMsg; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; +import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; @@ -377,6 +378,21 @@ public class ProtoUtils { ); } + private static TransportProtos.DeviceDeletedMsgProto toProto(DeviceDeleteMsg msg) { + return TransportProtos.DeviceDeletedMsgProto.newBuilder() + .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) + .setDeviceIdMSB(msg.getDeviceId().getId().getMostSignificantBits()) + .setDeviceIdLSB(msg.getDeviceId().getId().getLeastSignificantBits()) + .build(); + } + + private static DeviceDeleteMsg fromProto(TransportProtos.DeviceDeletedMsgProto proto) { + return new DeviceDeleteMsg( + TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB()))); + } + public static TransportProtos.ToDeviceActorNotificationMsgProto toProto(ToDeviceActorNotificationMsg msg) { if (msg instanceof DeviceEdgeUpdateMsg) { DeviceEdgeUpdateMsg updateMsg = (DeviceEdgeUpdateMsg) msg; @@ -406,6 +422,10 @@ public class ProtoUtils { RemoveRpcActorMsg updateMsg = (RemoveRpcActorMsg) msg; TransportProtos.RemoveRpcActorMsgProto proto = toProto(updateMsg); return TransportProtos.ToDeviceActorNotificationMsgProto.newBuilder().setRemoveRpcActorMsg(proto).build(); + } else if (msg instanceof DeviceDeleteMsg) { + DeviceDeleteMsg updateMsg = (DeviceDeleteMsg) msg; + TransportProtos.DeviceDeletedMsgProto proto = toProto(updateMsg); + return TransportProtos.ToDeviceActorNotificationMsgProto.newBuilder().setDeviceAssignToTenantMsgProto(proto).build(); } return null; } @@ -425,6 +445,8 @@ public class ProtoUtils { return fromProto(proto.getFromDeviceRpcResponseMsg()); } else if (proto.hasRemoveRpcActorMsg()) { return fromProto(proto.getRemoveRpcActorMsg()); + } else if (proto.hasDeviceAssignToTenantMsgProto()) { + return fromProto(proto.getDeviceAssignToTenantMsgProto()); } return null; } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 5b1eb1ae4f..5c3b9076ff 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -83,7 +83,9 @@ public interface TbClusterService extends TbQueueClusterService { void onDeviceUpdated(Device device, Device old); - void onDeviceDeleted(Device device, TbQueueCallback callback); + void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback); + + void onDeviceAssignedToTenant(TenantId oldTenantId, Device device); void onResourceChange(TbResource resource, TbQueueCallback callback); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 6116411b2e..5000e3025f 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -1004,6 +1004,13 @@ message RemoveRpcActorMsgProto { int64 deviceIdLSB = 6; } +message DeviceDeletedMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; +} + message ToDeviceActorNotificationMsgProto { DeviceEdgeUpdateMsgProto deviceEdgeUpdateMsg = 1; DeviceNameOrTypeUpdateMsgProto deviceNameOrTypeMsg = 2; @@ -1012,6 +1019,7 @@ message ToDeviceActorNotificationMsgProto { ToDeviceRpcRequestActorMsgProto toDeviceRpcRequestMsg = 5; FromDeviceRpcResponseActorMsgProto fromDeviceRpcResponseMsg = 6; RemoveRpcActorMsgProto removeRpcActorMsg = 7; + DeviceDeletedMsgProto deviceAssignToTenantMsgProto = 8; } /** diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 0a39d8d51b..c3d3623d8e 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -93,6 +93,8 @@ public enum MsgType { DEVICE_NAME_OR_TYPE_UPDATE_TO_DEVICE_ACTOR_MSG, + DEVICE_DELETE_TO_DEVICE_ACTOR_MSG, + DEVICE_EDGE_UPDATE_TO_DEVICE_ACTOR_MSG, DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG, diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/rule/engine/DeviceDeleteMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/rule/engine/DeviceDeleteMsg.java new file mode 100644 index 0000000000..1bdab7409b --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/rule/engine/DeviceDeleteMsg.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.rule.engine; + +import lombok.Data; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; + +@Data +public class DeviceDeleteMsg implements ToDeviceActorNotificationMsg { + + private static final long serialVersionUID = 4679029228395462172L; + + private final TenantId tenantId; + private final DeviceId deviceId; + + @Override + public MsgType getMsgType() { + return MsgType.DEVICE_DELETE_TO_DEVICE_ACTOR_MSG; + } +} From deb19d24c0b4ebdcf8fdf07c5e94d6397fb413ab Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Nov 2023 10:57:27 +0100 Subject: [PATCH 04/33] added verification for the corresponding test --- .../controller/DeviceControllerTest.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index 778a11a552..b6afaa34e1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -51,7 +51,6 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmSeverity; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.CustomerId; @@ -73,7 +72,9 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; +import org.thingsboard.server.service.state.DeviceStateService; import java.util.ArrayList; import java.util.List; @@ -82,6 +83,9 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -106,6 +110,9 @@ public class DeviceControllerTest extends AbstractControllerTest { @SpyBean private GatewayNotificationsService gatewayNotificationsService; + @SpyBean + private DeviceStateService deviceStateService; + @Autowired private DeviceDao deviceDao; @@ -1340,6 +1347,28 @@ public class DeviceControllerTest extends AbstractControllerTest { ActionType.ASSIGNED_TO_TENANT, savedDifferentTenant.getId().getId().toString(), savedDifferentTenant.getTitle()); testNotificationUpdateGatewayNever(); + Mockito.verify(deviceStateService, times(1)).onQueueMsg( + argThat(proto -> + proto.getTenantIdMSB() == savedTenant.getUuidId().getMostSignificantBits() && + proto.getTenantIdLSB() == savedTenant.getUuidId().getLeastSignificantBits() && + proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && + proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && + proto.getDeleted() + ), + any() + ); + + Mockito.verify(deviceStateService, times(1)).onQueueMsg( + argThat(proto -> + proto.getTenantIdMSB() == savedDifferentTenant.getUuidId().getMostSignificantBits() && + proto.getTenantIdLSB() == savedDifferentTenant.getUuidId().getLeastSignificantBits() && + proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && + proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && + proto.getAdded() + ), + any() + ); + login("tenant9@thingsboard.org", "testPassword1"); Device foundDevice1 = doGet("/api/device/" + assignedDevice.getId().getId(), Device.class); From 5de58644e550618a36d1ebec80995c1d32f36a4d Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Nov 2023 11:32:04 +0100 Subject: [PATCH 05/33] refactoring --- .../server/service/entitiy/device/DefaultTbDeviceService.java | 2 -- .../org/thingsboard/server/controller/DeviceControllerTest.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java index a3d04f6344..ac2d656ba6 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java @@ -232,8 +232,6 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T Tenant tenant = tenantService.findTenantById(tenantId); Device assignedDevice = deviceService.assignDeviceToTenant(newTenantId, device); - - notificationEntityService.notifyAssignDeviceToTenant(tenantId, newTenantId, deviceId, assignedDevice.getCustomerId(), assignedDevice, tenant, user, newTenantId.toString(), newTenant.getName()); diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index b6afaa34e1..cefe24e935 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -72,7 +72,6 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.state.DeviceStateService; @@ -85,7 +84,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; From deaf06970a59f1083af2b7eb214b2ac8de9cdf52 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Wed, 15 Nov 2023 07:56:06 +0200 Subject: [PATCH 06/33] Adapted tests to match new amount of dashboards for tenant --- .../controller/DashboardControllerTest.java | 21 +++++++++++++------ .../server/controller/HomePageApiTest.java | 2 +- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java index 6f0d6f9465..fb8064d6d5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java @@ -327,7 +327,18 @@ public class DashboardControllerTest extends AbstractControllerTest { @Test public void testFindTenantDashboards() throws Exception { - List dashboards = new ArrayList<>(); + List expectedDashboards = new ArrayList<>(); + PageLink pageLink = new PageLink(24); + PageData pageData = null; + do { + pageData = doGetTypedWithPageLink("/api/tenant/dashboards?", + new TypeReference>() { + }, pageLink); + expectedDashboards.addAll(pageData.getData()); + if (pageData.hasNext()) { + pageLink = pageLink.nextPageLink(); + } + } while (pageData.hasNext()); Mockito.reset(tbClusterService, auditLogService); @@ -335,7 +346,7 @@ public class DashboardControllerTest extends AbstractControllerTest { for (int i = 0; i < cntEntity; i++) { Dashboard dashboard = new Dashboard(); dashboard.setTitle("Dashboard" + i); - dashboards.add(new DashboardInfo(doPost("/api/dashboard", dashboard, Dashboard.class))); + expectedDashboards.add(new DashboardInfo(doPost("/api/dashboard", dashboard, Dashboard.class))); } testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new Dashboard(), new Dashboard(), @@ -343,8 +354,6 @@ public class DashboardControllerTest extends AbstractControllerTest { ActionType.ADDED, cntEntity, cntEntity, cntEntity); List loadedDashboards = new ArrayList<>(); - PageLink pageLink = new PageLink(24); - PageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/dashboards?", new TypeReference>() { @@ -355,10 +364,10 @@ public class DashboardControllerTest extends AbstractControllerTest { } } while (pageData.hasNext()); - dashboards.sort(idComparator); + expectedDashboards.sort(idComparator); loadedDashboards.sort(idComparator); - Assert.assertEquals(dashboards, loadedDashboards); + Assert.assertEquals(expectedDashboards, loadedDashboards); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java b/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java index d020fef1cb..c498b73815 100644 --- a/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java @@ -408,7 +408,7 @@ public class HomePageApiTest extends AbstractControllerTest { Assert.assertEquals(2, usageInfo.getUsers()); Assert.assertEquals(configuration.getMaxUsers(), usageInfo.getMaxUsers()); - Assert.assertEquals(0, usageInfo.getDashboards()); + Assert.assertEquals(1, usageInfo.getDashboards()); Assert.assertEquals(configuration.getMaxDashboards(), usageInfo.getMaxDashboards()); Assert.assertEquals(0, usageInfo.getTransportMessages()); From ebd50555eff87a0b799d0c5b72a7f68e460ac8c0 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Wed, 15 Nov 2023 17:29:47 +0200 Subject: [PATCH 07/33] Updated test --- .../org/thingsboard/server/controller/HomePageApiTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java b/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java index c498b73815..148c4afea9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java @@ -92,6 +92,8 @@ public class HomePageApiTest extends AbstractControllerTest { @MockBean private SmsService smsService; + private static final int DEFAULT_DASHBOARDS_COUNT = 1; + //For system administrator @Test public void testTenantsCountWsCmd() throws Exception { @@ -408,7 +410,7 @@ public class HomePageApiTest extends AbstractControllerTest { Assert.assertEquals(2, usageInfo.getUsers()); Assert.assertEquals(configuration.getMaxUsers(), usageInfo.getMaxUsers()); - Assert.assertEquals(1, usageInfo.getDashboards()); + Assert.assertEquals(DEFAULT_DASHBOARDS_COUNT, usageInfo.getDashboards()); Assert.assertEquals(configuration.getMaxDashboards(), usageInfo.getMaxDashboards()); Assert.assertEquals(0, usageInfo.getTransportMessages()); @@ -478,7 +480,8 @@ public class HomePageApiTest extends AbstractControllerTest { } usageInfo = doGet("/api/usage", UsageInfo.class); - Assert.assertEquals(dashboards.size(), usageInfo.getDashboards()); + int expectedDashboardsCount = dashboards.size() + DEFAULT_DASHBOARDS_COUNT; + Assert.assertEquals(expectedDashboardsCount, usageInfo.getDashboards()); } private Long getInitialEntityCount(EntityType entityType) throws Exception { From e9c003f76daf68c2c6998921bf74f71dfc670ba7 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 15 Nov 2023 15:14:41 +0200 Subject: [PATCH 08/33] Subscribe first in testSubscribingToUnreadNotificationsCount (cherry picked from commit edcc507f4b9ec40ee92c70f2c0d8f8490f7ccbe8) --- .../server/service/notification/NotificationApiTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java index 26be4e9bd3..ba0b4e4763 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java @@ -113,15 +113,13 @@ public class NotificationApiTest extends AbstractNotificationApiTest { @Test public void testSubscribingToUnreadNotificationsCount() { + wsClient.subscribeForUnreadNotificationsCount().waitForReply(true); NotificationTarget notificationTarget = createNotificationTarget(customerUserId); String notificationText1 = "Notification 1"; submitNotificationRequest(notificationTarget.getId(), notificationText1); String notificationText2 = "Notification 2"; submitNotificationRequest(notificationTarget.getId(), notificationText2); - wsClient.subscribeForUnreadNotificationsCount(); - wsClient.waitForReply(true); - await().atMost(2, TimeUnit.SECONDS) .until(() -> wsClient.getLastCountUpdate().getTotalUnreadCount() == 2); } From 80fc2d9b862f09b900fb3e0d2b7ce71ba98759a7 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 16 Nov 2023 15:06:00 +0100 Subject: [PATCH 09/33] improvements --- .../server/actors/tenant/TenantActor.java | 11 +++++++---- .../server/service/queue/ProtoUtils.java | 16 ++++++++-------- common/cluster-api/src/main/proto/queue.proto | 4 ++-- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index f508d967c9..86f338e89d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -228,7 +228,7 @@ public class TenantActor extends RuleChainManagerActor { log.warn("RECEIVED INVALID MESSAGE: {}", msg); } if (deletedDevices.contains(msg.getDeviceId())) { - log.warn("RECEIVED MESSAGE FOR DELETED DEVICE: {}", msg); + log.debug("RECEIVED MESSAGE FOR DELETED DEVICE: {}", msg); return; } TbActorRef deviceActor = getOrCreateDeviceActor(msg.getDeviceId()); @@ -250,7 +250,8 @@ public class TenantActor extends RuleChainManagerActor { log.info("[{}] Received API state update. Going to ENABLE Rule Engine execution.", tenantId); initRuleChains(); } - } else if (msg.getEntityId().getEntityType() == EntityType.EDGE) { + } + if (msg.getEntityId().getEntityType() == EntityType.EDGE) { EdgeId edgeId = new EdgeId(msg.getEntityId().getId()); EdgeRpcService edgeRpcService = systemContext.getEdgeRpcService(); if (msg.getEvent() == ComponentLifecycleEvent.DELETED) { @@ -259,11 +260,13 @@ public class TenantActor extends RuleChainManagerActor { Edge edge = systemContext.getEdgeService().findEdgeById(tenantId, edgeId); edgeRpcService.updateEdge(tenantId, edge); } - } else if (msg.getEntityId().getEntityType() == EntityType.DEVICE && ComponentLifecycleEvent.DELETED == msg.getEvent()) { + } + if (msg.getEntityId().getEntityType() == EntityType.DEVICE && ComponentLifecycleEvent.DELETED == msg.getEvent()) { DeviceId deviceId = (DeviceId) msg.getEntityId(); onToDeviceActorMsg(new DeviceDeleteMsg(tenantId, deviceId), true); deletedDevices.add(deviceId); - } else if (isRuleEngine) { + } + if (isRuleEngine) { TbActorRef target = getEntityActorRef(msg.getEntityId()); if (target != null) { if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java index 8986ca8b51..c93af1bb07 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java @@ -44,9 +44,9 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg; import org.thingsboard.server.common.msg.rpc.RemoveRpcActorMsg; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos; @@ -378,8 +378,8 @@ public class ProtoUtils { ); } - private static TransportProtos.DeviceDeletedMsgProto toProto(DeviceDeleteMsg msg) { - return TransportProtos.DeviceDeletedMsgProto.newBuilder() + private static TransportProtos.DeviceDeleteMsgProto toProto(DeviceDeleteMsg msg) { + return TransportProtos.DeviceDeleteMsgProto.newBuilder() .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) .setDeviceIdMSB(msg.getDeviceId().getId().getMostSignificantBits()) @@ -387,7 +387,7 @@ public class ProtoUtils { .build(); } - private static DeviceDeleteMsg fromProto(TransportProtos.DeviceDeletedMsgProto proto) { + private static DeviceDeleteMsg fromProto(TransportProtos.DeviceDeleteMsgProto proto) { return new DeviceDeleteMsg( TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB()))); @@ -424,8 +424,8 @@ public class ProtoUtils { return TransportProtos.ToDeviceActorNotificationMsgProto.newBuilder().setRemoveRpcActorMsg(proto).build(); } else if (msg instanceof DeviceDeleteMsg) { DeviceDeleteMsg updateMsg = (DeviceDeleteMsg) msg; - TransportProtos.DeviceDeletedMsgProto proto = toProto(updateMsg); - return TransportProtos.ToDeviceActorNotificationMsgProto.newBuilder().setDeviceAssignToTenantMsgProto(proto).build(); + TransportProtos.DeviceDeleteMsgProto proto = toProto(updateMsg); + return TransportProtos.ToDeviceActorNotificationMsgProto.newBuilder().setDeviceDeleteMsg(proto).build(); } return null; } @@ -445,8 +445,8 @@ public class ProtoUtils { return fromProto(proto.getFromDeviceRpcResponseMsg()); } else if (proto.hasRemoveRpcActorMsg()) { return fromProto(proto.getRemoveRpcActorMsg()); - } else if (proto.hasDeviceAssignToTenantMsgProto()) { - return fromProto(proto.getDeviceAssignToTenantMsgProto()); + } else if (proto.hasDeviceDeleteMsg()) { + return fromProto(proto.getDeviceDeleteMsg()); } return null; } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 5000e3025f..c8919cb5c7 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -1004,7 +1004,7 @@ message RemoveRpcActorMsgProto { int64 deviceIdLSB = 6; } -message DeviceDeletedMsgProto { +message DeviceDeleteMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; int64 deviceIdMSB = 3; @@ -1019,7 +1019,7 @@ message ToDeviceActorNotificationMsgProto { ToDeviceRpcRequestActorMsgProto toDeviceRpcRequestMsg = 5; FromDeviceRpcResponseActorMsgProto fromDeviceRpcResponseMsg = 6; RemoveRpcActorMsgProto removeRpcActorMsg = 7; - DeviceDeletedMsgProto deviceAssignToTenantMsgProto = 8; + DeviceDeleteMsgProto deviceDeleteMsg = 8; } /** From 9abfd128aed4663115d77bf518d9c38671d6ea78 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Nov 2023 13:35:12 +0100 Subject: [PATCH 10/33] fixed fluky tests --- .../server/msa/connectivity/MqttClientTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java index 5a7e9b631d..6ad50196b3 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java @@ -26,6 +26,7 @@ import io.netty.buffer.Unpooled; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -337,8 +338,12 @@ public class MqttClientTest extends AbstractContainerTest { MqttClient mqttClient = getMqttClient(deviceCredentials, listener); testRestClient.deleteDeviceIfExists(device.getId()); - TimeUnit.SECONDS.sleep(3 * timeoutMultiplier); - assertThat(mqttClient.isConnected()).isFalse(); + + Awaitility + .await() + .alias("Check device connection.") + .atMost(10, TimeUnit.SECONDS) + .until(() -> !mqttClient.isConnected()); } @Test From a17a17ee973a9a04b672d2695a75761115999dff Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Nov 2023 13:36:38 +0100 Subject: [PATCH 11/33] remove cassandra volume for blackbox --- .../org/thingsboard/server/msa/ThingsBoardDbInstaller.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index fe467ec341..7d87351bef 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -253,12 +253,16 @@ public class ThingsBoardDbInstaller { .add(tbVcExecutorLogVolume) .add(resolveRedisComposeVolumeLog()); + if (IS_HYBRID_MODE) { + rmVolumesCommand.add(cassandraDataVolume); + } + dockerCompose.withCommand(rmVolumesCommand.toString()); } private String resolveRedisComposeVolumeLog() { if (IS_REDIS_CLUSTER) { - return IntStream.range(0, 6).mapToObj(i -> redisClusterDataVolume + "-" + i).collect(Collectors.joining()); + return IntStream.range(0, 6).mapToObj(i -> " " + redisClusterDataVolume + "-" + i).collect(Collectors.joining()); } if (IS_REDIS_SENTINEL) { return redisSentinelDataVolume + "-" + "master " + " " + From 336161e1a2101ad442570021699fa15c73319089 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Fri, 17 Nov 2023 17:36:22 +0200 Subject: [PATCH 12/33] Added fix for docker command for gateway launch command and support for ipv6 adresses in device connectivity --- .../DeviceConnectivityController.java | 2 +- .../DeviceConnectivityControllerTest.java | 271 ++++++++++++------ .../device/DeviceConnectivityServiceImpl.java | 20 +- .../dao/util/DeviceConnectivityUtil.java | 50 +++- 4 files changed, 235 insertions(+), 108 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceConnectivityController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceConnectivityController.java index 83d4d16034..dca113621c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceConnectivityController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceConnectivityController.java @@ -106,7 +106,7 @@ public class DeviceConnectivityController extends BaseController { @RequestMapping(value = "/device-connectivity/gateway-launch/{deviceId}", method = RequestMethod.GET) @ResponseBody public JsonNode getGatewayLaunchCommands(@ApiParam(value = DEVICE_ID_PARAM_DESCRIPTION) - @PathVariable(DEVICE_ID) String strDeviceId, HttpServletRequest request) throws ThingsboardException, URISyntaxException { + @PathVariable(DEVICE_ID) String strDeviceId, HttpServletRequest request) throws ThingsboardException, URISyntaxException { checkParameter(DEVICE_ID, strDeviceId); DeviceId deviceId = new DeviceId(toUUID(strDeviceId)); Device device = checkDeviceId(deviceId, Operation.READ_CREDENTIALS); diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index 227d37f483..b629408855 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -18,8 +18,6 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -32,7 +30,6 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -46,7 +43,6 @@ import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileCon import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; @@ -58,14 +54,16 @@ import java.nio.file.Path; import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.CA_ROOT_CERT_PEM; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.COAP; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.COAPS; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.DOCKER; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.HTTP; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.HTTPS; +import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.LINUX; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTT; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTTS; -import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.CA_ROOT_CERT_PEM; +import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.WINDOWS; @TestPropertySource(properties = { "device.connectivity.mqtts.pem_cert_file=/tmp/" + CA_ROOT_CERT_PEM @@ -294,6 +292,140 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { DEVICE_TELEMETRY_TOPIC, credentials.getCredentialsId())); } + @Test + public void testFetchGatewayLaunchCommands() throws Exception { + Device device = new Device(); + device.setName("My device"); + device.setType("default"); + ObjectNode additionalInfo = JacksonUtil.newObjectNode(); + additionalInfo.put("gateway", true); + device.setAdditionalInfo(additionalInfo); + Device savedDevice = doPost("/api/device", device, Device.class); + DeviceCredentials credentials = + doGet("/api/device/" + savedDevice.getId().getId() + "/credentials", DeviceCredentials.class); + + JsonNode commands = + doGetTyped("/api/device-connectivity/gateway-launch/" + savedDevice.getId().getId(), new TypeReference<>() { + }); + + JsonNode dockerMqttCommands = commands.get(MQTT); + assertThat(dockerMqttCommands.get(LINUX).asText()).isEqualTo(String.format("docker run -it -v ~/.tb-gateway/logs:/thingsboard_gateway/logs -v ~/.tb-gateway/extensions:/thingsboard_gateway/extensions -v ~/.tb-gateway/config:/thingsboard_gateway/config --network=host -p 5000:5000 --name tbGatewayLocalhost -e host=localhost -e port=1883 -e accessToken=%s --restart always thingsboard/tb-gateway", credentials.getCredentialsId())); + assertThat(dockerMqttCommands.get(WINDOWS).asText()).isEqualTo("docker run -it -v %HOMEPATH%/tb-gateway/logs:/thingsboard_gateway/logs -v %HOMEPATH%/tb-gateway/extensions:/thingsboard_gateway/extensions -v %HOMEPATH%/tb-gateway/config:/thingsboard_gateway/config --network=host -p 5000:5000 --name tbGatewayLocalhost -e host=localhost -e port=1883 -e accessToken=" + credentials.getCredentialsId() + " --restart always thingsboard/tb-gateway"); + + JsonNode dockerMqttsCommands = commands.get(MQTTS); + assertThat(dockerMqttsCommands.get(LINUX).asText()).isEqualTo(String.format("docker run -it -v ~/.tb-gateway/logs:/thingsboard_gateway/logs -v ~/.tb-gateway/extensions:/thingsboard_gateway/extensions -v ~/.tb-gateway/config:/thingsboard_gateway/config --network=host -p 5000:5000 --name tbGatewayLocalhost -e host=localhost -e port=8883 -e accessToken=%s --restart always thingsboard/tb-gateway", credentials.getCredentialsId())); + assertThat(dockerMqttsCommands.get(WINDOWS).asText()).isEqualTo("docker run -it -v %HOMEPATH%/tb-gateway/logs:/thingsboard_gateway/logs -v %HOMEPATH%/tb-gateway/extensions:/thingsboard_gateway/extensions -v %HOMEPATH%/tb-gateway/config:/thingsboard_gateway/config --network=host -p 5000:5000 --name tbGatewayLocalhost -e host=localhost -e port=8883 -e accessToken=" + credentials.getCredentialsId() + " --restart always thingsboard/tb-gateway"); + } + + @Test + public void testFetchPublishTelemetryCommandsForDeviceWithIpV6LocalhostAddress() throws Exception { + loginSysAdmin(); + setConnectivityHost("::1"); + loginTenantAdmin(); + + Device device = new Device(); + device.setName("My device"); + device.setType("default"); + Device savedDevice = doPost("/api/device", device, Device.class); + DeviceCredentials credentials = + doGet("/api/device/" + savedDevice.getId().getId() + "/credentials", DeviceCredentials.class); + + JsonNode commands = + doGetTyped("/api/device-connectivity/" + savedDevice.getId().getId(), new TypeReference<>() { + }); + + assertThat(commands).hasSize(3); + JsonNode httpCommands = commands.get(HTTP); + assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://localhost:8080/api/v1/%s/telemetry " + + "--header Content-Type:application/json --data \"{temperature:25}\"", + credentials.getCredentialsId())); + assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://localhost/api/v1/%s/telemetry " + + "--header Content-Type:application/json --data \"{temperature:25}\"", + credentials.getCredentialsId())); + + JsonNode mqttCommands = commands.get(MQTT); + assertThat(mqttCommands.get(MQTT).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 -h localhost -p 1883 -t v1/devices/me/telemetry " + + "-u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(mqttCommands.get(MQTTS).get(0).asText()).isEqualTo("curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download"); + assertThat(mqttCommands.get(MQTTS).get(1).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 --cafile ca-root.pem -h localhost -p 8883 " + + "-t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + + JsonNode dockerMqttCommands = commands.get(MQTT).get(DOCKER); + assertThat(dockerMqttCommands.get(MQTT).asText()).isEqualTo(String.format("docker run --rm -it --network=host thingsboard/mosquitto-clients mosquitto_pub -d -q 1 -h localhost" + + " -p 1883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(dockerMqttCommands.get(MQTTS).asText()).isEqualTo(String.format("docker run --rm -it --network=host thingsboard/mosquitto-clients " + + "/bin/sh -c \"curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download && " + + "mosquitto_pub -d -q 1 --cafile ca-root.pem -h localhost -p 8883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"\"", + credentials.getCredentialsId())); + + JsonNode linuxCoapCommands = commands.get(COAP); + assertThat(linuxCoapCommands.get(COAP).asText()).isEqualTo(String.format("coap-client -v 6 -m POST coap://localhost:5683/api/v1/%s/telemetry " + + "-t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(linuxCoapCommands.get(COAPS).asText()).isEqualTo(String.format("coap-client-openssl -v 6 -m POST coaps://localhost:5684/api/v1/%s/telemetry" + + " -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + + JsonNode dockerCoapCommands = commands.get(COAP).get(DOCKER); + assertThat(dockerCoapCommands.get(COAP).asText()).isEqualTo(String.format("docker run --rm -it --network=host" + + " thingsboard/coap-clients coap-client -v 6 -m POST coap://localhost:5683/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(dockerCoapCommands.get(COAPS).asText()).isEqualTo(String.format("docker run --rm -it --network=host" + + " thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://localhost:5684/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + } + + @Test + public void testFetchPublishTelemetryCommandsForDeviceWithIpV6Address() throws Exception { + + loginSysAdmin(); + setConnectivityHost("1:1:1:1:1:1:1:1"); + loginTenantAdmin(); + + Device device = new Device(); + device.setName("My device"); + device.setType("default"); + Device savedDevice = doPost("/api/device", device, Device.class); + DeviceCredentials credentials = + doGet("/api/device/" + savedDevice.getId().getId() + "/credentials", DeviceCredentials.class); + + JsonNode commands = + doGetTyped("/api/device-connectivity/" + savedDevice.getId().getId(), new TypeReference<>() { + }); + assertThat(commands).hasSize(3); + JsonNode httpCommands = commands.get(HTTP); + assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://[1:1:1:1:1:1:1:1]:8080/api/v1/%s/telemetry " + + "--header Content-Type:application/json --data \"{temperature:25}\"", + credentials.getCredentialsId())); + assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://[1:1:1:1:1:1:1:1]/api/v1/%s/telemetry " + + "--header Content-Type:application/json --data \"{temperature:25}\"", + credentials.getCredentialsId())); + + JsonNode mqttCommands = commands.get(MQTT); + assertThat(mqttCommands.get(MQTT).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 -h 1:1:1:1:1:1:1:1 -p 1883 -t v1/devices/me/telemetry " + + "-u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(mqttCommands.get(MQTTS).get(0).asText()).isEqualTo("curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download"); + assertThat(mqttCommands.get(MQTTS).get(1).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 --cafile ca-root.pem -h 1:1:1:1:1:1:1:1 -p 8883 " + + "-t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + + JsonNode dockerMqttCommands = commands.get(MQTT).get(DOCKER); + assertThat(dockerMqttCommands.get(MQTT).asText()).isEqualTo(String.format("docker run --rm -it thingsboard/mosquitto-clients mosquitto_pub -d -q 1 -h 1:1:1:1:1:1:1:1" + + " -p 1883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(dockerMqttCommands.get(MQTTS).asText()).isEqualTo(String.format("docker run --rm -it thingsboard/mosquitto-clients " + + "/bin/sh -c \"curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download && " + + "mosquitto_pub -d -q 1 --cafile ca-root.pem -h 1:1:1:1:1:1:1:1 -p 8883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"\"", + credentials.getCredentialsId())); + + JsonNode linuxCoapCommands = commands.get(COAP); + assertThat(linuxCoapCommands.get(COAP).asText()).isEqualTo(String.format("coap-client -v 6 -m POST coap://[1:1:1:1:1:1:1:1]:5683/api/v1/%s/telemetry " + + "-t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(linuxCoapCommands.get(COAPS).asText()).isEqualTo(String.format("coap-client-openssl -v 6 -m POST coaps://[1:1:1:1:1:1:1:1]:5684/api/v1/%s/telemetry" + + " -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + + JsonNode dockerCoapCommands = commands.get(COAP).get(DOCKER); + assertThat(dockerCoapCommands.get(COAP).asText()).isEqualTo(String.format("docker run --rm -it" + + " thingsboard/coap-clients coap-client -v 6 -m POST coap://[1:1:1:1:1:1:1:1]:5683/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(dockerCoapCommands.get(COAPS).asText()).isEqualTo(String.format("docker run --rm -it" + + " thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://[1:1:1:1:1:1:1:1]:5684/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + + } + @Test public void testFetchPublishTelemetryCommandsForDeviceWithMqttBasicCreds() throws Exception { Device device = new Device(); @@ -449,47 +581,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { public void testFetchPublishTelemetryCommandsForDefaultDeviceIfPortsSetToDefault() throws Exception { loginSysAdmin(); - ObjectNode config = JacksonUtil.newObjectNode(); - - ObjectNode http = JacksonUtil.newObjectNode(); - http.put("enabled", true); - http.put("host", ""); - http.put("port", 80); - config.set("http", http); - - ObjectNode https = JacksonUtil.newObjectNode(); - https.put("enabled", true); - https.put("host", ""); - https.put("port", 443); - config.set("https", https); - - ObjectNode mqtt = JacksonUtil.newObjectNode(); - mqtt.put("enabled", false); - mqtt.put("host", ""); - mqtt.put("port", 1883); - config.set("mqtt", mqtt); - - ObjectNode mqtts = JacksonUtil.newObjectNode(); - mqtts.put("enabled", false); - mqtts.put("host", ""); - mqtts.put("port", 8883); - config.set("mqtts", mqtts); - - ObjectNode coap = JacksonUtil.newObjectNode(); - coap.put("enabled", false); - coap.put("host", ""); - coap.put("port", 5683); - config.set("coap", coap); - - ObjectNode coaps = JacksonUtil.newObjectNode(); - coaps.put("enabled", false); - coaps.put("host", ""); - coaps.put("port", 5684); - config.set("coaps", coaps); - - AdminSettings adminSettings = doGet("/api/admin/settings/connectivity", AdminSettings.class); - adminSettings.setJsonValue(config); - doPost("/api/admin/settings", adminSettings).andExpect(status().isOk()); + setConnectivityHost(""); login("tenant2@thingsboard.org", "testPassword1"); @@ -518,47 +610,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { public void testFetchPublishTelemetryCommandsForDefaultDeviceIfHostIsNotLocalhost() throws Exception { loginSysAdmin(); - ObjectNode config = JacksonUtil.newObjectNode(); - - ObjectNode http = JacksonUtil.newObjectNode(); - http.put("enabled", true); - http.put("host", "test.domain"); - http.put("port", 8080); - config.set("http", http); - - ObjectNode https = JacksonUtil.newObjectNode(); - https.put("enabled", true); - https.put("host", "test.domain"); - https.put("port", 443); - config.set("https", https); - - ObjectNode mqtt = JacksonUtil.newObjectNode(); - mqtt.put("enabled", true); - mqtt.put("host", "test.domain"); - mqtt.put("port", 1883); - config.set("mqtt", mqtt); - - ObjectNode mqtts = JacksonUtil.newObjectNode(); - mqtts.put("enabled", true); - mqtts.put("host", "test.domain"); - mqtts.put("port", 8883); - config.set("mqtts", mqtts); - - ObjectNode coap = JacksonUtil.newObjectNode(); - coap.put("enabled", true); - coap.put("host", "test.domain"); - coap.put("port", 5683); - config.set("coap", coap); - - ObjectNode coaps = JacksonUtil.newObjectNode(); - coaps.put("enabled", true); - coaps.put("host", "test.domain"); - coaps.put("port", 5684); - config.set("coaps", coaps); - - AdminSettings adminSettings = doGet("/api/admin/settings/connectivity", AdminSettings.class); - adminSettings.setJsonValue(config); - doPost("/api/admin/settings", adminSettings).andExpect(status().isOk()); + setConnectivityHost("test.domain"); login("tenant2@thingsboard.org", "testPassword1"); @@ -612,4 +664,49 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { assertThat(dockerCoapCommands.get(COAPS).asText()).isEqualTo(String.format("docker run --rm -it " + "thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://test.domain:5684/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); } + + + private void setConnectivityHost(String host) throws Exception { + ObjectNode config = JacksonUtil.newObjectNode(); + + ObjectNode http = JacksonUtil.newObjectNode(); + http.put("enabled", true); + http.put("host", host); + http.put("port", 8080); + config.set("http", http); + + ObjectNode https = JacksonUtil.newObjectNode(); + https.put("enabled", true); + https.put("host", host); + https.put("port", 443); + config.set("https", https); + + ObjectNode mqtt = JacksonUtil.newObjectNode(); + mqtt.put("enabled", true); + mqtt.put("host", host); + mqtt.put("port", 1883); + config.set("mqtt", mqtt); + + ObjectNode mqtts = JacksonUtil.newObjectNode(); + mqtts.put("enabled", true); + mqtts.put("host", host); + mqtts.put("port", 8883); + config.set("mqtts", mqtts); + + ObjectNode coap = JacksonUtil.newObjectNode(); + coap.put("enabled", true); + coap.put("host", host); + coap.put("port", 5683); + config.set("coap", coap); + + ObjectNode coaps = JacksonUtil.newObjectNode(); + coaps.put("enabled", true); + coaps.put("host", host); + coaps.put("port", 5684); + config.set("coaps", coaps); + + AdminSettings adminSettings = doGet("/api/admin/settings/connectivity", AdminSettings.class); + adminSettings.setJsonValue(config); + doPost("/api/admin/settings", adminSettings).andExpect(status().isOk()); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java index 9f67093529..58f2f352d7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java @@ -43,7 +43,6 @@ import org.thingsboard.server.dao.util.DeviceConnectivityUtil; import java.io.InputStream; import java.io.InputStreamReader; -import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -64,6 +63,7 @@ import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.LINUX; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTT; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTTS; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.WINDOWS; +import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.getHost; @Service("DeviceConnectivityDaoService") @Slf4j @@ -230,7 +230,7 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService deviceCredentials.getCredentialsType() != DeviceCredentialsType.ACCESS_TOKEN) { return null; } - String hostName = getHost(baseUrl, properties); + String hostName = getHost(baseUrl, properties, protocol); String propertiesPort = properties.getPort(); String port = (propertiesPort.isEmpty() || HTTP_DEFAULT_PORT.equals(propertiesPort) || HTTPS_DEFAULT_PORT.equals(propertiesPort)) ? "" : ":" + propertiesPort; @@ -278,14 +278,14 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService private String getMqttPublishCommand(String baseUrl, String deviceTelemetryTopic, DeviceCredentials deviceCredentials) throws URISyntaxException { DeviceConnectivityInfo properties = getConnectivity(MQTT); - String mqttHost = getHost(baseUrl, properties); + String mqttHost = getHost(baseUrl, properties, MQTT); String mqttPort = properties.getPort().isEmpty() ? null : properties.getPort(); return DeviceConnectivityUtil.getMqttPublishCommand(MQTT, mqttHost, mqttPort, deviceTelemetryTopic, deviceCredentials); } private List getMqttsPublishCommand(String baseUrl, String deviceTelemetryTopic, DeviceCredentials deviceCredentials) throws URISyntaxException { DeviceConnectivityInfo properties = getConnectivity(MQTTS); - String mqttHost = getHost(baseUrl, properties); + String mqttHost = getHost(baseUrl, properties, MQTTS); String mqttPort = properties.getPort().isEmpty() ? null : properties.getPort(); String pubCommand = DeviceConnectivityUtil.getMqttPublishCommand(MQTTS, mqttHost, mqttPort, deviceTelemetryTopic, deviceCredentials); @@ -301,7 +301,7 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService private JsonNode getGatewayDockerCommands(String baseUrl, DeviceCredentials deviceCredentials, String mqttType) throws URISyntaxException { ObjectNode dockerLaunchCommands = JacksonUtil.newObjectNode(); DeviceConnectivityInfo properties = getConnectivity(mqttType); - String mqttHost = getHost(baseUrl, properties); + String mqttHost = getHost(baseUrl, properties, mqttType); String mqttPort = properties.getPort().isEmpty() ? null : properties.getPort(); Optional.ofNullable(DeviceConnectivityUtil.getGatewayLaunchCommand(LINUX, mqttHost, mqttPort, deviceCredentials)) .ifPresent(v -> dockerLaunchCommands.put(LINUX, v)); @@ -312,7 +312,7 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService private String getDockerMqttPublishCommand(String protocol, String baseUrl, String deviceTelemetryTopic, DeviceCredentials deviceCredentials) throws URISyntaxException { DeviceConnectivityInfo properties = getConnectivity(protocol); - String mqttHost = getHost(baseUrl, properties); + String mqttHost = getHost(baseUrl, properties, protocol); String mqttPort = properties.getPort().isEmpty() ? null : properties.getPort(); return DeviceConnectivityUtil.getDockerMqttPublishCommand(protocol, baseUrl, mqttHost, mqttPort, deviceTelemetryTopic, deviceCredentials); } @@ -352,20 +352,16 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService private String getCoapPublishCommand(String protocol, String baseUrl, DeviceCredentials deviceCredentials) throws URISyntaxException { DeviceConnectivityInfo properties = getConnectivity(protocol); - String hostName = getHost(baseUrl, properties); + String hostName = getHost(baseUrl, properties, protocol); String port = properties.getPort().isEmpty() ? "" : ":" + properties.getPort(); return DeviceConnectivityUtil.getCoapPublishCommand(protocol, hostName, port, deviceCredentials); } private String getDockerCoapPublishCommand(String protocol, String baseUrl, DeviceCredentials deviceCredentials) throws URISyntaxException { DeviceConnectivityInfo properties = getConnectivity(protocol); - String host = getHost(baseUrl, properties); + String host = getHost(baseUrl, properties, protocol); String port = properties.getPort().isEmpty() ? "" : ":" + properties.getPort(); return DeviceConnectivityUtil.getDockerCoapPublishCommand(protocol, host, port, deviceCredentials); } - private String getHost(String baseUrl, DeviceConnectivityInfo properties) throws URISyntaxException { - return properties.getHost().isEmpty() ? new URI(baseUrl).getHost() : properties.getHost(); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index aa446c12b5..f905a9d288 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -19,9 +19,14 @@ import org.apache.commons.lang3.StringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.dao.device.DeviceConnectivityInfo; -import java.util.Arrays; -import java.util.List; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.util.regex.Pattern; public class DeviceConnectivityUtil { @@ -41,7 +46,7 @@ public class DeviceConnectivityUtil { public static final String GATEWAY_DOCKER_RUN = "docker run -it "; public static final String MQTT_IMAGE = "thingsboard/mosquitto-clients "; public static final String COAP_IMAGE = "thingsboard/coap-clients "; - public static final List LOCAL_HOSTS = Arrays.asList("localhost", "127.0.0.1"); + private final static Pattern VALID_URL_PATTERN = Pattern.compile("^(http|https|ftp)://.*$"); public static String getHttpPublishCommand(String protocol, String host, String port, DeviceCredentials deviceCredentials) { return String.format("curl -v -X POST %s://%s%s/api/v1/%s/telemetry --header Content-Type:application/json --data " + JSON_EXAMPLE_PAYLOAD, @@ -71,7 +76,7 @@ public class DeviceConnectivityUtil { command.append(" -u \"").append(credentials.getUserName()).append("\""); } if (credentials.getPassword() != null) { - command.append(" -P \"").append(credentials.getPassword()).append("\"");; + command.append(" -P \"").append(credentials.getPassword()).append("\""); } } else { return null; @@ -90,17 +95,19 @@ public class DeviceConnectivityUtil { gatewayVolumePathPrefix = "%HOMEPATH%/tb-gateway"; } - String gatewayContainerName = "tbGateway" + StringUtils.capitalize(host.replace(".", "")); + String gatewayContainerName = "tbGateway" + StringUtils.capitalize(host.replaceAll("[^A-Za-z0-9]", "")); StringBuilder command = new StringBuilder(GATEWAY_DOCKER_RUN); command.append("-v {gatewayVolumePathPrefix}/logs:/thingsboard_gateway/logs".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); command.append(" -v {gatewayVolumePathPrefix}/extensions:/thingsboard_gateway/extensions".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); command.append(" -v {gatewayVolumePathPrefix}/config:/thingsboard_gateway/config".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); + command.append(isLocalhost(host) ? " --network=host" : ""); + command.append(" -p 5000:5000"); command.append(" --name ").append(gatewayContainerName); command.append(" -e host=").append(host); command.append(" -e port=").append(port); - switch(deviceCredentials.getCredentialsType()) { + switch (deviceCredentials.getCredentialsType()) { case ACCESS_TOKEN: command.append(" -e accessToken=").append(deviceCredentials.getCredentialsId()); break; @@ -139,7 +146,7 @@ public class DeviceConnectivityUtil { } StringBuilder mqttDockerCommand = new StringBuilder(); - mqttDockerCommand.append(DOCKER_RUN).append(LOCAL_HOSTS.contains(host) ? "--network=host ":"").append(MQTT_IMAGE); + mqttDockerCommand.append(DOCKER_RUN).append(isLocalhost(host) ? "--network=host " : "").append(MQTT_IMAGE); if (MQTTS.equals(protocol)) { mqttDockerCommand.append("/bin/sh -c \"") @@ -171,6 +178,33 @@ public class DeviceConnectivityUtil { public static String getDockerCoapPublishCommand(String protocol, String host, String port, DeviceCredentials deviceCredentials) { String coapCommand = getCoapPublishCommand(protocol, host, port, deviceCredentials); - return coapCommand != null ? String.format("%s%s%s", DOCKER_RUN + (LOCAL_HOSTS.contains(host) ? "--network=host ":""), COAP_IMAGE, coapCommand) : null; + return coapCommand != null ? String.format("%s%s%s", DOCKER_RUN + (isLocalhost(host) ? "--network=host " : ""), COAP_IMAGE, coapCommand) : null; + } + + public static String getHost(String baseUrl, DeviceConnectivityInfo properties, String protocol) throws URISyntaxException { + String host = properties.getHost().isEmpty() ? baseUrl : properties.getHost(); + if (VALID_URL_PATTERN.matcher(host).matches()) { + host = new URI(host).getHost(); + } else { + try { + InetAddress inetAddress = InetAddress.getByName(host); + host = inetAddress.getCanonicalHostName().replaceAll("[\\[\\]]", ""); + if (inetAddress instanceof Inet6Address && !inetAddress.isLoopbackAddress() && !protocol.startsWith(MQTT)) { + host = "[" + host + "]"; + } + } catch (UnknownHostException e) { + return host; + } + } + return host; + } + + private static boolean isLocalhost(String host) { + try { + InetAddress inetAddress = InetAddress.getByName(host); + return inetAddress.isLoopbackAddress(); + } catch (UnknownHostException e) { + return false; + } } } From 9a14260bf1d42d83882d452606a2e6e86263bf29 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Fri, 17 Nov 2023 18:54:01 +0200 Subject: [PATCH 13/33] Host check refactoring --- .../DeviceConnectivityControllerTest.java | 42 ++++++++++++++++++- .../dao/util/DeviceConnectivityUtil.java | 21 ++++++---- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index b629408855..4ca59dc843 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -581,7 +581,47 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { public void testFetchPublishTelemetryCommandsForDefaultDeviceIfPortsSetToDefault() throws Exception { loginSysAdmin(); - setConnectivityHost(""); + ObjectNode config = JacksonUtil.newObjectNode(); + + ObjectNode http = JacksonUtil.newObjectNode(); + http.put("enabled", true); + http.put("host", ""); + http.put("port", 80); + config.set("http", http); + + ObjectNode https = JacksonUtil.newObjectNode(); + https.put("enabled", true); + https.put("host", ""); + https.put("port", 443); + config.set("https", https); + + ObjectNode mqtt = JacksonUtil.newObjectNode(); + mqtt.put("enabled", false); + mqtt.put("host", ""); + mqtt.put("port", 1883); + config.set("mqtt", mqtt); + + ObjectNode mqtts = JacksonUtil.newObjectNode(); + mqtts.put("enabled", false); + mqtts.put("host", ""); + mqtts.put("port", 8883); + config.set("mqtts", mqtts); + + ObjectNode coap = JacksonUtil.newObjectNode(); + coap.put("enabled", false); + coap.put("host", ""); + coap.put("port", 5683); + config.set("coap", coap); + + ObjectNode coaps = JacksonUtil.newObjectNode(); + coaps.put("enabled", false); + coaps.put("host", ""); + coaps.put("port", 5684); + config.set("coaps", coaps); + + AdminSettings adminSettings = doGet("/api/admin/settings/connectivity", AdminSettings.class); + adminSettings.setJsonValue(config); + doPost("/api/admin/settings", adminSettings).andExpect(status().isOk()); login("tenant2@thingsboard.org", "testPassword1"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index f905a9d288..d1eb434100 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -183,17 +183,20 @@ public class DeviceConnectivityUtil { public static String getHost(String baseUrl, DeviceConnectivityInfo properties, String protocol) throws URISyntaxException { String host = properties.getHost().isEmpty() ? baseUrl : properties.getHost(); + InetAddress inetAddress; if (VALID_URL_PATTERN.matcher(host).matches()) { host = new URI(host).getHost(); - } else { - try { - InetAddress inetAddress = InetAddress.getByName(host); - host = inetAddress.getCanonicalHostName().replaceAll("[\\[\\]]", ""); - if (inetAddress instanceof Inet6Address && !inetAddress.isLoopbackAddress() && !protocol.startsWith(MQTT)) { - host = "[" + host + "]"; - } - } catch (UnknownHostException e) { - return host; + } + try { + inetAddress = InetAddress.getByName(host); + host = inetAddress.getHostName(); + } catch (UnknownHostException e) { + return host; + } + if (inetAddress instanceof Inet6Address) { + host = host.replaceAll("[\\[\\]]", ""); + if (!inetAddress.isLoopbackAddress() && !protocol.startsWith(MQTT)) { + host = "[" + host + "]"; } } return host; From d196e69b23be90617c78826cdddb6fa2642484f1 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 20 Nov 2023 09:11:52 +0200 Subject: [PATCH 14/33] Updated regex to check url --- .../org/thingsboard/server/dao/util/DeviceConnectivityUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index d1eb434100..151932af1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -46,7 +46,7 @@ public class DeviceConnectivityUtil { public static final String GATEWAY_DOCKER_RUN = "docker run -it "; public static final String MQTT_IMAGE = "thingsboard/mosquitto-clients "; public static final String COAP_IMAGE = "thingsboard/coap-clients "; - private final static Pattern VALID_URL_PATTERN = Pattern.compile("^(http|https|ftp)://.*$"); + private final static Pattern VALID_URL_PATTERN = Pattern.compile("^(https?)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"); public static String getHttpPublishCommand(String protocol, String host, String port, DeviceCredentials deviceCredentials) { return String.format("curl -v -X POST %s://%s%s/api/v1/%s/telemetry --header Content-Type:application/json --data " + JSON_EXAMPLE_PAYLOAD, From 8777f1297240470aa238ce3231c88fc25212f860 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 20 Nov 2023 09:47:25 +0200 Subject: [PATCH 15/33] Added null safety if user set a bad ipv6 address --- .../server/dao/util/DeviceConnectivityUtil.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index 151932af1a..44ddb095aa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -182,12 +182,17 @@ public class DeviceConnectivityUtil { } public static String getHost(String baseUrl, DeviceConnectivityInfo properties, String protocol) throws URISyntaxException { - String host = properties.getHost().isEmpty() ? baseUrl : properties.getHost(); + String initialHost = properties.getHost().isEmpty() ? baseUrl : properties.getHost(); InetAddress inetAddress; - if (VALID_URL_PATTERN.matcher(host).matches()) { - host = new URI(host).getHost(); + String host = null; + if (VALID_URL_PATTERN.matcher(initialHost).matches()) { + host = new URI(initialHost).getHost(); + } + if (host == null) { + host = initialHost; } try { + host = host.replaceAll("^https?://", ""); inetAddress = InetAddress.getByName(host); host = inetAddress.getHostName(); } catch (UnknownHostException e) { From 5e8858d5637ad1801761c2de5998379ed3c6ef8b Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 20 Nov 2023 11:17:19 +0200 Subject: [PATCH 16/33] Updated host check logic and related test --- .../DeviceConnectivityControllerTest.java | 20 +++++++++---------- .../dao/util/DeviceConnectivityUtil.java | 3 +-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index 4ca59dc843..f517d2e9c5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -336,39 +336,39 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { assertThat(commands).hasSize(3); JsonNode httpCommands = commands.get(HTTP); - assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://localhost:8080/api/v1/%s/telemetry " + + assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://[::1]:8080/api/v1/%s/telemetry " + "--header Content-Type:application/json --data \"{temperature:25}\"", credentials.getCredentialsId())); - assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://localhost/api/v1/%s/telemetry " + + assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://[::1]/api/v1/%s/telemetry " + "--header Content-Type:application/json --data \"{temperature:25}\"", credentials.getCredentialsId())); JsonNode mqttCommands = commands.get(MQTT); - assertThat(mqttCommands.get(MQTT).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 -h localhost -p 1883 -t v1/devices/me/telemetry " + + assertThat(mqttCommands.get(MQTT).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 -h ::1 -p 1883 -t v1/devices/me/telemetry " + "-u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); assertThat(mqttCommands.get(MQTTS).get(0).asText()).isEqualTo("curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download"); - assertThat(mqttCommands.get(MQTTS).get(1).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 --cafile ca-root.pem -h localhost -p 8883 " + + assertThat(mqttCommands.get(MQTTS).get(1).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 --cafile ca-root.pem -h ::1 -p 8883 " + "-t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); JsonNode dockerMqttCommands = commands.get(MQTT).get(DOCKER); - assertThat(dockerMqttCommands.get(MQTT).asText()).isEqualTo(String.format("docker run --rm -it --network=host thingsboard/mosquitto-clients mosquitto_pub -d -q 1 -h localhost" + + assertThat(dockerMqttCommands.get(MQTT).asText()).isEqualTo(String.format("docker run --rm -it --network=host thingsboard/mosquitto-clients mosquitto_pub -d -q 1 -h ::1" + " -p 1883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); assertThat(dockerMqttCommands.get(MQTTS).asText()).isEqualTo(String.format("docker run --rm -it --network=host thingsboard/mosquitto-clients " + "/bin/sh -c \"curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download && " + - "mosquitto_pub -d -q 1 --cafile ca-root.pem -h localhost -p 8883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"\"", + "mosquitto_pub -d -q 1 --cafile ca-root.pem -h ::1 -p 8883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"\"", credentials.getCredentialsId())); JsonNode linuxCoapCommands = commands.get(COAP); - assertThat(linuxCoapCommands.get(COAP).asText()).isEqualTo(String.format("coap-client -v 6 -m POST coap://localhost:5683/api/v1/%s/telemetry " + + assertThat(linuxCoapCommands.get(COAP).asText()).isEqualTo(String.format("coap-client -v 6 -m POST coap://[::1]:5683/api/v1/%s/telemetry " + "-t json -e \"{temperature:25}\"", credentials.getCredentialsId())); - assertThat(linuxCoapCommands.get(COAPS).asText()).isEqualTo(String.format("coap-client-openssl -v 6 -m POST coaps://localhost:5684/api/v1/%s/telemetry" + + assertThat(linuxCoapCommands.get(COAPS).asText()).isEqualTo(String.format("coap-client-openssl -v 6 -m POST coaps://[::1]:5684/api/v1/%s/telemetry" + " -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); JsonNode dockerCoapCommands = commands.get(COAP).get(DOCKER); assertThat(dockerCoapCommands.get(COAP).asText()).isEqualTo(String.format("docker run --rm -it --network=host" + - " thingsboard/coap-clients coap-client -v 6 -m POST coap://localhost:5683/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + " thingsboard/coap-clients coap-client -v 6 -m POST coap://[::1]:5683/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); assertThat(dockerCoapCommands.get(COAPS).asText()).isEqualTo(String.format("docker run --rm -it --network=host" + - " thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://localhost:5684/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + " thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://[::1]:5684/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); } @Test diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index 44ddb095aa..efd75e305e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -194,13 +194,12 @@ public class DeviceConnectivityUtil { try { host = host.replaceAll("^https?://", ""); inetAddress = InetAddress.getByName(host); - host = inetAddress.getHostName(); } catch (UnknownHostException e) { return host; } if (inetAddress instanceof Inet6Address) { host = host.replaceAll("[\\[\\]]", ""); - if (!inetAddress.isLoopbackAddress() && !protocol.startsWith(MQTT)) { + if (!protocol.startsWith(MQTT)) { host = "[" + host + "]"; } } From 8a926ae239820181778314c071e8f605fd63dbbf Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 20 Nov 2023 17:25:19 +0200 Subject: [PATCH 17/33] Changes, due to comments --- .../dao/util/DeviceConnectivityUtil.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index efd75e305e..3434b47965 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -44,6 +44,8 @@ public class DeviceConnectivityUtil { public static final String JSON_EXAMPLE_PAYLOAD = "\"{temperature:25}\""; public static final String DOCKER_RUN = "docker run --rm -it "; public static final String GATEWAY_DOCKER_RUN = "docker run -it "; + + public static final String NETWORK_HOST_PARAM = "--network=host "; public static final String MQTT_IMAGE = "thingsboard/mosquitto-clients "; public static final String COAP_IMAGE = "thingsboard/coap-clients "; private final static Pattern VALID_URL_PATTERN = Pattern.compile("^(https?)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"); @@ -98,14 +100,14 @@ public class DeviceConnectivityUtil { String gatewayContainerName = "tbGateway" + StringUtils.capitalize(host.replaceAll("[^A-Za-z0-9]", "")); StringBuilder command = new StringBuilder(GATEWAY_DOCKER_RUN); - command.append("-v {gatewayVolumePathPrefix}/logs:/thingsboard_gateway/logs".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); - command.append(" -v {gatewayVolumePathPrefix}/extensions:/thingsboard_gateway/extensions".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); - command.append(" -v {gatewayVolumePathPrefix}/config:/thingsboard_gateway/config".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); - command.append(isLocalhost(host) ? " --network=host" : ""); - command.append(" -p 5000:5000"); - command.append(" --name ").append(gatewayContainerName); - command.append(" -e host=").append(host); - command.append(" -e port=").append(port); + command.append("-v {gatewayVolumePathPrefix}/logs:/thingsboard_gateway/logs ".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); + command.append("-v {gatewayVolumePathPrefix}/extensions:/thingsboard_gateway/extensions ".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); + command.append("-v {gatewayVolumePathPrefix}/config:/thingsboard_gateway/config ".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); + command.append(isLocalhost(host) ? NETWORK_HOST_PARAM : ""); + command.append("-p 5000:5000 "); + command.append("--name ").append(gatewayContainerName).append(" "); + command.append("-e host=").append(host).append(" "); + command.append("-e port=").append(port); switch (deviceCredentials.getCredentialsType()) { case ACCESS_TOKEN: @@ -146,7 +148,7 @@ public class DeviceConnectivityUtil { } StringBuilder mqttDockerCommand = new StringBuilder(); - mqttDockerCommand.append(DOCKER_RUN).append(isLocalhost(host) ? "--network=host " : "").append(MQTT_IMAGE); + mqttDockerCommand.append(DOCKER_RUN).append(isLocalhost(host) ? NETWORK_HOST_PARAM : "").append(MQTT_IMAGE); if (MQTTS.equals(protocol)) { mqttDockerCommand.append("/bin/sh -c \"") @@ -178,7 +180,7 @@ public class DeviceConnectivityUtil { public static String getDockerCoapPublishCommand(String protocol, String host, String port, DeviceCredentials deviceCredentials) { String coapCommand = getCoapPublishCommand(protocol, host, port, deviceCredentials); - return coapCommand != null ? String.format("%s%s%s", DOCKER_RUN + (isLocalhost(host) ? "--network=host " : ""), COAP_IMAGE, coapCommand) : null; + return coapCommand != null ? String.format("%s%s%s", DOCKER_RUN + (isLocalhost(host) ? NETWORK_HOST_PARAM : ""), COAP_IMAGE, coapCommand) : null; } public static String getHost(String baseUrl, DeviceConnectivityInfo properties, String protocol) throws URISyntaxException { @@ -199,7 +201,7 @@ public class DeviceConnectivityUtil { } if (inetAddress instanceof Inet6Address) { host = host.replaceAll("[\\[\\]]", ""); - if (!protocol.startsWith(MQTT)) { + if (!MQTT.equals(protocol) && !MQTTS.equals(protocol)) { host = "[" + host + "]"; } } From 31aa1d25e030243dd2599913396e3f148699d274 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 10 Oct 2023 16:22:57 +0200 Subject: [PATCH 18/33] added fst stats --- .../server/service/stats/FstStats.java | 29 +++++ .../src/main/resources/thingsboard.yml | 6 +- .../server/common/data/FSTUtils.java | 122 +++++++++++++++++- 3 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/stats/FstStats.java diff --git a/application/src/main/java/org/thingsboard/server/service/stats/FstStats.java b/application/src/main/java/org/thingsboard/server/service/stats/FstStats.java new file mode 100644 index 0000000000..d66b9fd138 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/stats/FstStats.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.stats; + +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.FSTUtils; + +@Service +public class FstStats { + + @Scheduled(initialDelayString = "${fst.stats.printInterval}", fixedDelayString = "${fst.stats.printInterval}") + public void printStats() { + FSTUtils.printStats(); + } +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ecc89c84c5..bba7800a28 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1680,4 +1680,8 @@ management: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' \ No newline at end of file + include: '${METRICS_ENDPOINTS_EXPOSE:info}' + +fst: + stats: + printInterval: "${FST_STATS_PRINT_INTERVAL:600000}" # 10 min by default \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/FSTUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/FSTUtils.java index 83447d7a6b..e70c6f29c6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/FSTUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/FSTUtils.java @@ -15,21 +15,137 @@ */ package org.thingsboard.server.common.data; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + @Slf4j public class FSTUtils { public static final FSTConfiguration CONFIG = FSTConfiguration.createDefaultConfiguration(); + private static final ConcurrentHashMap STATS = new ConcurrentHashMap<>(); + @SuppressWarnings("unchecked") public static T decode(byte[] byteArray) { - return byteArray != null && byteArray.length > 0 ? (T) CONFIG.asObject(byteArray) : null; + long startTime = System.nanoTime(); + T result = byteArray != null && byteArray.length > 0 ? (T) CONFIG.asObject(byteArray) : null; + long endTime = System.nanoTime(); + + if (log.isDebugEnabled() && result != null) { + String className = result.getClass().getSimpleName(); + STATS.computeIfAbsent(className, k -> new Stats()).incrementDecode(endTime - startTime); + } + + return result; + } + + public static byte[] encode(T msg) { + long startTime = System.nanoTime(); + byte[] result = CONFIG.asByteArray(msg); + long endTime = System.nanoTime(); + + if (log.isDebugEnabled() && msg != null) { + String className = msg.getClass().getSimpleName(); + STATS.computeIfAbsent(className, k -> new Stats()).incrementEncode(endTime - startTime); + } + + return result; + } + + public static void printStats() { + if (log.isDebugEnabled()) { + List topDecode = STATS.entrySet().stream() + .filter(e -> e.getValue().getAvgDecodeTime() > 0) + .sorted((e1, e2) -> Long.compare(e2.getValue().getAvgDecodeTime(), e1.getValue().getAvgDecodeTime())) + .limit(5) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + + List topEncode = STATS.entrySet().stream() + .filter(e -> e.getValue().getAvgEncodeTime() > 0) + .sorted((e1, e2) -> Long.compare(e2.getValue().getAvgEncodeTime(), e1.getValue().getAvgEncodeTime())) + .limit(5) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + + List topDecodeCount = STATS.entrySet().stream() + .filter(e -> e.getValue().getDecodeCount().get() > 0) + .sorted((e1, e2) -> Long.compare(e2.getValue().getDecodeCount().get(), e1.getValue().getDecodeCount().get())) + .limit(5) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + + List topEncodeCount = STATS.entrySet().stream() + .filter(e -> e.getValue().getEncodeCount().get() > 0) + .sorted((e1, e2) -> Long.compare(e2.getValue().getEncodeCount().get(), e1.getValue().getEncodeCount().get())) + .limit(5) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + + for (Map.Entry entry : STATS.entrySet()) { + Stats stats = entry.getValue(); + if (stats.isNotEmpty()) { + log.debug("[FST stats] [{}] {}", entry.getKey(), stats); + stats.reset(); + } + } + + log.debug("[FST stats] Top 5 slowest 'decode' {}", topDecode); + log.debug("[FST stats] Top 5 slowest 'encode' {}", topEncode); + log.debug("[FST stats] Top 5 'decode' count {}", topDecodeCount); + log.debug("[FST stats] Top 5 'encode' count {}", topEncodeCount); + } } - public static byte[] encode(T msq) { - return CONFIG.asByteArray(msq); + @Data + private static class Stats { + private final AtomicLong encodeCount = new AtomicLong(); + private final AtomicLong decodeCount = new AtomicLong(); + private final AtomicLong totalEncodeTime = new AtomicLong(); + private final AtomicLong totalDecodeTime = new AtomicLong(); + + private void incrementEncode(long time) { + encodeCount.incrementAndGet(); + totalEncodeTime.addAndGet(time); + } + + private void incrementDecode(long time) { + decodeCount.incrementAndGet(); + totalDecodeTime.addAndGet(time); + } + + private boolean isNotEmpty() { + return encodeCount.get() > 0 || decodeCount.get() > 0; + } + + private long getAvgEncodeTime() { + long count = encodeCount.get(); + return count > 0 ? totalEncodeTime.get() / count : 0; + } + + private long getAvgDecodeTime() { + long count = decodeCount.get(); + return count > 0 ? totalDecodeTime.get() / count : 0; + } + + private void reset() { + encodeCount.set(0); + decodeCount.set(0); + totalEncodeTime.set(0); + totalDecodeTime.set(0); + } + + @Override + public String toString() { + return String.format("decodeCount [%d] avgDecodeTime [%d] encodedCount [%d] avgEncodeTime [%d]", decodeCount.get(), getAvgDecodeTime(), encodeCount.get(), getAvgEncodeTime()); + } } } From d6b56177a27dd3cf139a5f8f73ea2da3eab0b800 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 21 Nov 2023 16:00:30 +0100 Subject: [PATCH 19/33] fst stats in Grafana --- .../src/main/resources/thingsboard.yml | 6 +- .../cache/RedisTbTransactionalCache.java | 12 +- .../server/common/data/FSTUtils.java | 122 +----------------- .../server/common/data/FstStatsService.java | 15 +-- .../queue/util/ProtoWithFSTService.java | 13 +- .../common/stats/FstStatsServiceImpl.java | 42 ++++++ 6 files changed, 73 insertions(+), 137 deletions(-) rename application/src/main/java/org/thingsboard/server/service/stats/FstStats.java => common/data/src/main/java/org/thingsboard/server/common/data/FstStatsService.java (59%) create mode 100644 common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index bba7800a28..ecc89c84c5 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1680,8 +1680,4 @@ management: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' - -fst: - stats: - printInterval: "${FST_STATS_PRINT_INTERVAL:600000}" # 10 min by default \ No newline at end of file + include: '${METRICS_ENDPOINTS_EXPOSE:info}' \ No newline at end of file diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index c5af954392..c2dd66dde0 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -17,6 +17,7 @@ package org.thingsboard.server.cache; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.support.NullValue; import org.springframework.data.redis.connection.RedisConnection; import org.springframework.data.redis.connection.RedisConnectionFactory; @@ -27,6 +28,7 @@ import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; import org.springframework.data.redis.core.types.Expiration; import org.springframework.data.redis.serializer.RedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; +import org.thingsboard.server.common.data.FstStatsService; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.util.JedisClusterCRC16; @@ -44,6 +46,9 @@ public abstract class RedisTbTransactionalCache STATS = new ConcurrentHashMap<>(); - @SuppressWarnings("unchecked") public static T decode(byte[] byteArray) { - long startTime = System.nanoTime(); - T result = byteArray != null && byteArray.length > 0 ? (T) CONFIG.asObject(byteArray) : null; - long endTime = System.nanoTime(); - - if (log.isDebugEnabled() && result != null) { - String className = result.getClass().getSimpleName(); - STATS.computeIfAbsent(className, k -> new Stats()).incrementDecode(endTime - startTime); - } - - return result; - } - - public static byte[] encode(T msg) { - long startTime = System.nanoTime(); - byte[] result = CONFIG.asByteArray(msg); - long endTime = System.nanoTime(); - - if (log.isDebugEnabled() && msg != null) { - String className = msg.getClass().getSimpleName(); - STATS.computeIfAbsent(className, k -> new Stats()).incrementEncode(endTime - startTime); - } - - return result; - } - - public static void printStats() { - if (log.isDebugEnabled()) { - List topDecode = STATS.entrySet().stream() - .filter(e -> e.getValue().getAvgDecodeTime() > 0) - .sorted((e1, e2) -> Long.compare(e2.getValue().getAvgDecodeTime(), e1.getValue().getAvgDecodeTime())) - .limit(5) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - - List topEncode = STATS.entrySet().stream() - .filter(e -> e.getValue().getAvgEncodeTime() > 0) - .sorted((e1, e2) -> Long.compare(e2.getValue().getAvgEncodeTime(), e1.getValue().getAvgEncodeTime())) - .limit(5) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - - List topDecodeCount = STATS.entrySet().stream() - .filter(e -> e.getValue().getDecodeCount().get() > 0) - .sorted((e1, e2) -> Long.compare(e2.getValue().getDecodeCount().get(), e1.getValue().getDecodeCount().get())) - .limit(5) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - - List topEncodeCount = STATS.entrySet().stream() - .filter(e -> e.getValue().getEncodeCount().get() > 0) - .sorted((e1, e2) -> Long.compare(e2.getValue().getEncodeCount().get(), e1.getValue().getEncodeCount().get())) - .limit(5) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - - for (Map.Entry entry : STATS.entrySet()) { - Stats stats = entry.getValue(); - if (stats.isNotEmpty()) { - log.debug("[FST stats] [{}] {}", entry.getKey(), stats); - stats.reset(); - } - } - - log.debug("[FST stats] Top 5 slowest 'decode' {}", topDecode); - log.debug("[FST stats] Top 5 slowest 'encode' {}", topEncode); - log.debug("[FST stats] Top 5 'decode' count {}", topDecodeCount); - log.debug("[FST stats] Top 5 'encode' count {}", topEncodeCount); - } + return byteArray != null && byteArray.length > 0 ? (T) CONFIG.asObject(byteArray) : null; } - @Data - private static class Stats { - private final AtomicLong encodeCount = new AtomicLong(); - private final AtomicLong decodeCount = new AtomicLong(); - private final AtomicLong totalEncodeTime = new AtomicLong(); - private final AtomicLong totalDecodeTime = new AtomicLong(); - - private void incrementEncode(long time) { - encodeCount.incrementAndGet(); - totalEncodeTime.addAndGet(time); - } - - private void incrementDecode(long time) { - decodeCount.incrementAndGet(); - totalDecodeTime.addAndGet(time); - } - - private boolean isNotEmpty() { - return encodeCount.get() > 0 || decodeCount.get() > 0; - } - - private long getAvgEncodeTime() { - long count = encodeCount.get(); - return count > 0 ? totalEncodeTime.get() / count : 0; - } - - private long getAvgDecodeTime() { - long count = decodeCount.get(); - return count > 0 ? totalDecodeTime.get() / count : 0; - } - - private void reset() { - encodeCount.set(0); - decodeCount.set(0); - totalEncodeTime.set(0); - totalDecodeTime.set(0); - } - - @Override - public String toString() { - return String.format("decodeCount [%d] avgDecodeTime [%d] encodedCount [%d] avgEncodeTime [%d]", decodeCount.get(), getAvgDecodeTime(), encodeCount.get(), getAvgEncodeTime()); - } + public static byte[] encode(T msq) { + return CONFIG.asByteArray(msq); } } diff --git a/application/src/main/java/org/thingsboard/server/service/stats/FstStats.java b/common/data/src/main/java/org/thingsboard/server/common/data/FstStatsService.java similarity index 59% rename from application/src/main/java/org/thingsboard/server/service/stats/FstStats.java rename to common/data/src/main/java/org/thingsboard/server/common/data/FstStatsService.java index d66b9fd138..06f0d1a528 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/FstStats.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/FstStatsService.java @@ -13,17 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.stats; +package org.thingsboard.server.common.data; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.FSTUtils; +public interface FstStatsService { -@Service -public class FstStats { + void incrementEncode(Class clazz); + + void incrementDecode(Class clazz); - @Scheduled(initialDelayString = "${fst.stats.printInterval}", fixedDelayString = "${fst.stats.printInterval}") - public void printStats() { - FSTUtils.printStats(); - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java index b0d0b182e9..e35c86e070 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.util; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.FSTUtils; +import org.thingsboard.server.common.data.FstStatsService; import java.util.Optional; @@ -26,12 +28,17 @@ import java.util.Optional; @Service public class ProtoWithFSTService implements DataDecodingEncodingService { + @Autowired + private FstStatsService fstStatsService; + public static final FSTConfiguration CONFIG = FSTConfiguration.createDefaultConfiguration(); @Override public Optional decode(byte[] byteArray) { try { - return Optional.ofNullable(FSTUtils.decode(byteArray)); + Optional optional = Optional.ofNullable(FSTUtils.decode(byteArray)); + optional.ifPresent(obj -> fstStatsService.incrementDecode(obj.getClass())); + return optional; } catch (IllegalArgumentException e) { log.error("Error during deserialization message, [{}]", e.getMessage()); return Optional.empty(); @@ -41,7 +48,9 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { @Override public byte[] encode(T msq) { - return FSTUtils.encode(msq); + var bytes = FSTUtils.encode(msq); + fstStatsService.incrementEncode(msq.getClass()); + return bytes; } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java new file mode 100644 index 0000000000..5d78b8c36a --- /dev/null +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.stats; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.FstStatsService; + +import java.util.concurrent.ConcurrentHashMap; + +@Service +public class FstStatsServiceImpl implements FstStatsService { + private final ConcurrentHashMap encodeCounters = new ConcurrentHashMap<>(); + private final ConcurrentHashMap decodeCounters = new ConcurrentHashMap<>(); + + @Autowired + private StatsFactory statsFactory; + + @Override + public void incrementEncode(Class clazz) { + encodeCounters.computeIfAbsent(clazz.getSimpleName(), key -> statsFactory.createStatsCounter("fstEncode", key)).increment(); + } + + @Override + public void incrementDecode(Class clazz) { + decodeCounters.computeIfAbsent(clazz.getSimpleName(), key -> statsFactory.createStatsCounter("fstDecode", key)).increment(); + } + +} From 3f2633fba55e718873712cfcf4d0ba7db19a157b Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 23 Nov 2023 12:22:15 +0200 Subject: [PATCH 20/33] extended widgetTypes access to customer user authority --- .../server/controller/WidgetTypeController.java | 4 ++-- .../controller/WidgetTypeControllerTest.java | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java index f90a196edf..0109a327e8 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java @@ -216,7 +216,7 @@ public class WidgetTypeController extends AutoCommitController { @ApiOperation(value = "Get all Widget types for specified Bundle (getBundleWidgetTypes)", notes = "Returns an array of Widget Type objects that belong to specified Widget Bundle." + WIDGET_TYPE_DESCRIPTION + " " + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/widgetTypes", params = {"widgetsBundleId"}, method = RequestMethod.GET) @ResponseBody public List getBundleWidgetTypes( @@ -248,7 +248,7 @@ public class WidgetTypeController extends AutoCommitController { @ApiOperation(value = "Get all Widget types details for specified Bundle (getBundleWidgetTypes)", notes = "Returns an array of Widget Type Details objects that belong to specified Widget Bundle." + WIDGET_TYPE_DETAILS_DESCRIPTION + " " + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/widgetTypesDetails", params = {"widgetsBundleId"}, method = RequestMethod.GET) @ResponseBody public List getBundleWidgetTypesDetails( diff --git a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java index 9cc9fd0596..553b0d6490 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java @@ -190,6 +190,20 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { Collections.sort(loadedWidgetTypes, idComparator); Assert.assertEquals(widgetTypes, loadedWidgetTypes); + + loginCustomerUser(); + + List loadedWidgetTypes2 = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", + new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + Collections.sort(loadedWidgetTypes2, idComparator); + Assert.assertEquals(widgetTypes, loadedWidgetTypes2); + + List loadedWidgetTypes3 = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", + new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + List widgetTypes3 = loadedWidgetTypes3.stream().map(WidgetType::new).collect(Collectors.toList()); + Collections.sort(widgetTypes3, idComparator); + Assert.assertEquals(widgetTypes3, loadedWidgetTypes); + } @Test From 248bcefa809f86ccf9aa6003419585f28c204df0 Mon Sep 17 00:00:00 2001 From: nick Date: Sat, 25 Nov 2023 19:39:58 +0200 Subject: [PATCH 21/33] stringToBytes: refactoring, Input parameter Object as HexString, DigString, Integer, Byte --- .../thingsboard/script/api/tbel/TbUtils.java | 18 +++++- .../script/api/tbel/TbUtilsTest.java | 57 +++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 31bd634221..07c2baee2d 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -182,7 +182,7 @@ public class TbUtils { return TbJson.parse(ctx, jsonStr); } - public static String bytesToString(List bytesList) { + public static String bytesToString(List bytesList) { byte[] bytes = bytesFromList(bytesList); return new String(bytes); } @@ -210,10 +210,22 @@ public class TbUtils { } } - private static byte[] bytesFromList(List bytesList) { + private static byte[] bytesFromList(List bytesList) { byte[] bytes = new byte[bytesList.size()]; for (int i = 0; i < bytesList.size(); i++) { - bytes[i] = bytesList.get(i); + if (bytesList.get(i) instanceof Integer) { + byte val = ((Integer) bytesList.get(i)).byteValue(); + if (((Integer) bytesList.get(i)).intValue() > 255 || ((Integer) bytesList.get(i)).intValue() < -128){ + throw new NumberFormatException("The value " + bytesList.get(i) + " could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255) !"); + } else { + bytes[i] = val; } + } else if (bytesList.get(i) instanceof String) { + bytes[i] = parseInt((String) bytesList.get(i)).byteValue(); + } else if (bytesList.get(i) instanceof Byte) { + bytes[i] = (byte) bytesList.get(i); + } else { + throw new NumberFormatException("Value \"" + bytesList.get(i) + " could not be converted to a byte. Must be format HexDecimal/String/Integer !"); + } } return bytes; } diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index d2900d6ec4..b80ac85ca7 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; import java.util.List; import java.util.Random; @@ -385,6 +386,62 @@ public class TbUtilsTest { Assert.assertThrows(IllegalAccessException.class, () -> TbUtils.stringToBytes(ctx, ((ExecutionHashMap) finalInputJson).get("hello"), "UTF-8")); } + @Test + public void bytesFromList() { + byte[] arrayBytes = {(byte)0x00, (byte)0x08, (byte)0x10, (byte)0x1C, (byte)0xFF, (byte)0xFC, (byte)0xAD, (byte)0x88, (byte)0x75, (byte)0x74, (byte)0x8A, (byte)0x82}; + Object[] arrayMix = { "0x00", 8, "16", "0x1C", 255, 252, 173, 136, 117, 116, -118, "-126"}; + + String expected = new String(arrayBytes); + ArrayList listBytes = new ArrayList<>(arrayBytes.length); + for (Byte element : arrayBytes) { + listBytes.add(element); + } + Assert.assertEquals(expected, TbUtils.bytesToString(listBytes)); + + ArrayList listMix = new ArrayList<>(arrayMix.length); + for (Object element : arrayMix) { + listMix.add(element); + } + Assert.assertEquals(expected, TbUtils.bytesToString(listMix)); + } + + @Test + public void bytesFromList_Error() { + List listHex = new ArrayList<>(); + listHex.add("0xFG"); + try { + TbUtils.bytesToString(listHex); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("Failed radix: [16] for value: \"FG\"!")); + } + listHex.add(0, "1F"); + try { + TbUtils.bytesToString(listHex); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("Failed radix: [10] for value: \"1F\"!")); + } + + ArrayList listIntBytes = new ArrayList<>(); + listIntBytes.add(-129); + try { + TbUtils.bytesToString(listIntBytes); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("The value -129 could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255")); + } + + listIntBytes.add(0, 256); + try { + TbUtils.bytesToString(listIntBytes); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("The value 256 could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255")); + } + } + + private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { From baa684f72b59b9a584a4ed8e003c14d30480ba64 Mon Sep 17 00:00:00 2001 From: nick Date: Sun, 26 Nov 2023 21:58:29 +0200 Subject: [PATCH 22/33] stringToBytes: refactoring test --- .../test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index b80ac85ca7..3a3fe22000 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -389,7 +389,7 @@ public class TbUtilsTest { @Test public void bytesFromList() { byte[] arrayBytes = {(byte)0x00, (byte)0x08, (byte)0x10, (byte)0x1C, (byte)0xFF, (byte)0xFC, (byte)0xAD, (byte)0x88, (byte)0x75, (byte)0x74, (byte)0x8A, (byte)0x82}; - Object[] arrayMix = { "0x00", 8, "16", "0x1C", 255, 252, 173, 136, 117, 116, -118, "-126"}; + Object[] arrayMix = { "0x00", 8, "16", "0x1C", 255, (byte)0xFC, 173, 136, 117, 116, -118, "-126"}; String expected = new String(arrayBytes); ArrayList listBytes = new ArrayList<>(arrayBytes.length); From 3ab4c8d468b2986d7884cd12694921991832fb95 Mon Sep 17 00:00:00 2001 From: nick Date: Sun, 26 Nov 2023 22:28:50 +0200 Subject: [PATCH 23/33] stringToBytes: refactoring --- .../src/main/java/org/thingsboard/script/api/tbel/TbUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 07c2baee2d..8fde4cd332 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -187,7 +187,7 @@ public class TbUtils { return new String(bytes); } - public static String bytesToString(List bytesList, String charsetName) throws UnsupportedEncodingException { + public static String bytesToString(List bytesList, String charsetName) throws UnsupportedEncodingException { byte[] bytes = bytesFromList(bytesList); return new String(bytes, charsetName); } From e8ba1e17ebfc3dfb6b74ca2bc14bb2018ac4ccae Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 27 Nov 2023 11:42:54 +0200 Subject: [PATCH 24/33] Monitoring for IPs associated with the domain --- .../monitoring/config/MonitoringTarget.java | 4 ++ .../transport/TransportMonitoringConfig.java | 3 +- .../transport/TransportMonitoringTarget.java | 1 + .../service/BaseMonitoringService.java | 39 ++++++++++++++++--- .../TransportsMonitoringService.java | 7 ++++ .../src/main/resources/tb-monitoring.yml | 16 ++++++-- 6 files changed, 59 insertions(+), 11 deletions(-) diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/config/MonitoringTarget.java b/monitoring/src/main/java/org/thingsboard/monitoring/config/MonitoringTarget.java index 0e62670f81..0176b14d54 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/config/MonitoringTarget.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/config/MonitoringTarget.java @@ -21,4 +21,8 @@ public interface MonitoringTarget { UUID getDeviceId(); + String getBaseUrl(); + + boolean isCheckDomainIps(); + } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringConfig.java b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringConfig.java index 77d702f779..c5f843192c 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringConfig.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringConfig.java @@ -23,9 +23,8 @@ import java.util.List; @Data public abstract class TransportMonitoringConfig implements MonitoringConfig { - private int requestTimeoutMs; - private List targets; + private int requestTimeoutMs; public abstract TransportType getTransportType(); diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java index 816f64fbce..e3277af0ff 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java @@ -25,6 +25,7 @@ public class TransportMonitoringTarget implements MonitoringTarget { private String baseUrl; private DeviceConfig device; // set manually during initialization + private boolean checkDomainIps; @Override public UUID getDeviceId() { diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java index def53557db..24ca9de4d4 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.monitoring.service; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; @@ -29,9 +30,14 @@ import org.thingsboard.monitoring.service.transport.TransportHealthChecker; import org.thingsboard.monitoring.util.TbStopWatch; import javax.annotation.PostConstruct; +import java.net.InetAddress; +import java.net.URI; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; +import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; @Slf4j public abstract class BaseMonitoringService, T extends MonitoringTarget> { @@ -60,15 +66,36 @@ public abstract class BaseMonitoringService, T ext tbClient.logIn(); configs.forEach(config -> { config.getTargets().forEach(target -> { - BaseHealthChecker healthChecker = (BaseHealthChecker) createHealthChecker(config, target); - log.info("Initializing {}", healthChecker.getClass().getSimpleName()); - healthChecker.initialize(tbClient); - devices.add(target.getDeviceId()); - healthCheckers.add(healthChecker); + initHealthChecker(target, config); + if (target.isCheckDomainIps()) { + initIpsHealthCheckers(target, config); + } }); }); } + private void initHealthChecker(T target, C config) { + BaseHealthChecker healthChecker = (BaseHealthChecker) createHealthChecker(config, target); + log.info("Initializing {} for {}", healthChecker.getClass().getSimpleName(), target.getBaseUrl()); + healthChecker.initialize(tbClient); + devices.add(target.getDeviceId()); + healthCheckers.add(healthChecker); + } + + @SneakyThrows + private void initIpsHealthCheckers(T target, C config) { + URI baseUrl = new URI(target.getBaseUrl()); + String domain = baseUrl.getHost(); + + Set ips = Arrays.stream(InetAddress.getAllByName(domain)) + .map(InetAddress::getHostAddress) + .collect(Collectors.toSet()); + for (String ip : ips) { + String url = new URI(baseUrl.getScheme(), null, ip, baseUrl.getPort(), "", null, null).toString(); + initHealthChecker(createTarget(url), config); + } + } + public final void runChecks() { if (healthCheckers.isEmpty()) { return; @@ -99,6 +126,8 @@ public abstract class BaseMonitoringService, T ext protected abstract BaseHealthChecker createHealthChecker(C config, T target); + protected abstract T createTarget(String baseUrl); + protected abstract String getName(); } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportsMonitoringService.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportsMonitoringService.java index b3ce86e799..ca7c5a94da 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportsMonitoringService.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportsMonitoringService.java @@ -33,6 +33,13 @@ public final class TransportsMonitoringService extends BaseMonitoringService Date: Mon, 27 Nov 2023 11:43:18 +0200 Subject: [PATCH 25/33] updated test to cover sysadmin authority --- .../controller/WidgetTypeControllerTest.java | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java index 553b0d6490..e515250321 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java @@ -193,17 +193,29 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { loginCustomerUser(); - List loadedWidgetTypes2 = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", + List loadedWidgetTypesCustomer = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); - Collections.sort(loadedWidgetTypes2, idComparator); - Assert.assertEquals(widgetTypes, loadedWidgetTypes2); + Collections.sort(loadedWidgetTypesCustomer, idComparator); + Assert.assertEquals(widgetTypes, loadedWidgetTypesCustomer); - List loadedWidgetTypes3 = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", + List customerLoadedWidgetTypesDetails = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); - List widgetTypes3 = loadedWidgetTypes3.stream().map(WidgetType::new).collect(Collectors.toList()); - Collections.sort(widgetTypes3, idComparator); - Assert.assertEquals(widgetTypes3, loadedWidgetTypes); + List widgetTypesFromDetailsListCustomer = customerLoadedWidgetTypesDetails.stream().map(WidgetType::new).collect(Collectors.toList()); + Collections.sort(widgetTypesFromDetailsListCustomer, idComparator); + Assert.assertEquals(widgetTypesFromDetailsListCustomer, loadedWidgetTypes); + loginSysAdmin(); + + List sysAdminLoadedWidgetTypes = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", + new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + Collections.sort(sysAdminLoadedWidgetTypes, idComparator); + Assert.assertEquals(widgetTypes, sysAdminLoadedWidgetTypes); + + List sysAdminLoadedWidgetTypesDetails = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", + new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + List widgetTypesFromDetailsListSysAdmin = sysAdminLoadedWidgetTypesDetails.stream().map(WidgetType::new).collect(Collectors.toList()); + Collections.sort(widgetTypesFromDetailsListSysAdmin, idComparator); + Assert.assertEquals(widgetTypesFromDetailsListSysAdmin, loadedWidgetTypes); } @Test From 24241010b72f6f4965d96b44be7239f99d7257db Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 27 Nov 2023 11:47:28 +0200 Subject: [PATCH 26/33] Root Rule Chain for monitoring tenant --- .../src/main/resources/root_rule_chain.json | 436 ++++++++++++++++++ 1 file changed, 436 insertions(+) create mode 100644 monitoring/src/main/resources/root_rule_chain.json diff --git a/monitoring/src/main/resources/root_rule_chain.json b/monitoring/src/main/resources/root_rule_chain.json new file mode 100644 index 0000000000..ed8a93fb63 --- /dev/null +++ b/monitoring/src/main/resources/root_rule_chain.json @@ -0,0 +1,436 @@ +{ + "ruleChain": { + "additionalInfo": null, + "name": "Root Rule Chain", + "type": "CORE", + "firstRuleNodeId": null, + "root": false, + "debugMode": false, + "configuration": null, + "externalId": null + }, + "metadata": { + "firstNodeIndex": 12, + "nodes": [ + { + "additionalInfo": { + "description": null, + "layoutX": 1202, + "layoutY": 221 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", + "name": "Save Timeseries", + "debugMode": true, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "defaultTTL": 0 + }, + "externalId": null + }, + { + "additionalInfo": { + "layoutX": 1000, + "layoutY": 167 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", + "name": "Save Attributes", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 1, + "configuration": { + "scope": "CLIENT_SCOPE", + "notifyDevice": false, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": false + }, + "externalId": null + }, + { + "additionalInfo": { + "layoutX": 566, + "layoutY": 302 + }, + "type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode", + "name": "Message Type Switch", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "version": 0 + }, + "externalId": null + }, + { + "additionalInfo": { + "layoutX": 1000, + "layoutY": 381 + }, + "type": "org.thingsboard.rule.engine.action.TbLogNode", + "name": "Log RPC from Device", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "scriptLang": "TBEL", + "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);", + "tbelScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" + }, + "externalId": null + }, + { + "additionalInfo": { + "layoutX": 1000, + "layoutY": 494 + }, + "type": "org.thingsboard.rule.engine.action.TbLogNode", + "name": "Log Other", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "scriptLang": "TBEL", + "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);", + "tbelScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" + }, + "externalId": null + }, + { + "additionalInfo": { + "layoutX": 1000, + "layoutY": 583 + }, + "type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode", + "name": "RPC Call Request", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "timeoutInSeconds": 60 + }, + "externalId": null + }, + { + "additionalInfo": { + "layoutX": 255, + "layoutY": 301 + }, + "type": "org.thingsboard.rule.engine.filter.TbOriginatorTypeFilterNode", + "name": "Is Entity Group", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "originatorTypes": [ + "ENTITY_GROUP" + ] + }, + "externalId": null + }, + { + "additionalInfo": { + "layoutX": 319, + "layoutY": 109 + }, + "type": "org.thingsboard.rule.engine.filter.TbMsgTypeFilterNode", + "name": "Post attributes or RPC request", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "messageTypes": [ + "POST_ATTRIBUTES_REQUEST", + "RPC_CALL_FROM_SERVER_TO_DEVICE" + ] + }, + "externalId": null + }, + { + "additionalInfo": { + "layoutX": 627, + "layoutY": 108 + }, + "type": "org.thingsboard.rule.engine.transform.TbDuplicateMsgToGroupNode", + "name": "Duplicate To Group Entities", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "entityGroupId": null, + "entityGroupIsMessageOriginator": true + }, + "externalId": null + }, + { + "additionalInfo": { + "description": "Process incoming messages from devices with the alarm rules defined in the device profile. Dispatch all incoming messages with \"Success\" relation type.", + "layoutX": 45, + "layoutY": 359 + }, + "type": "org.thingsboard.rule.engine.profile.TbDeviceProfileNode", + "name": "Device Profile Node", + "debugMode": true, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "persistAlarmRulesState": false, + "fetchAlarmRulesStateOnStart": false + }, + "externalId": null + }, + { + "additionalInfo": { + "description": "", + "layoutX": 160, + "layoutY": 631 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", + "name": "Test JS script", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "scriptLang": "JS", + "jsScript": "var test = {\n a: 'a',\n b: 'b'\n};\nreturn test.a === 'a' && test.b === 'b';", + "tbelScript": "return msg.temperature > 20;" + }, + "externalId": null + }, + { + "additionalInfo": { + "description": "", + "layoutX": 427, + "layoutY": 541 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", + "name": "Test TBEL script", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "scriptLang": "TBEL", + "jsScript": "return msg.temperature > 20;", + "tbelScript": "var a = \"a\";\nvar b = \"b\";\nreturn a.equals(\"a\") && b.equals(\"b\");" + }, + "externalId": null + }, + { + "additionalInfo": { + "description": "", + "layoutX": 40, + "layoutY": 252 + }, + "type": "org.thingsboard.rule.engine.transform.TbTransformMsgNode", + "name": "Add arrival timestamp", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "scriptLang": "TBEL", + "jsScript": "return {msg: msg, metadata: metadata, msgType: msgType};", + "tbelScript": "metadata.arrivalTs = Date.now();\nreturn {msg: msg, metadata: metadata, msgType: msgType};" + }, + "externalId": null + }, + { + "additionalInfo": { + "description": "", + "layoutX": 1467, + "layoutY": 267 + }, + "type": "org.thingsboard.rule.engine.transform.TbTransformMsgNode", + "name": "Calculate additional latencies", + "debugMode": true, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "scriptLang": "TBEL", + "jsScript": "return {msg: msg, metadata: metadata, msgType: msgType};", + "tbelScript": "var arrivalLatency = metadata.arrivalTs - metadata.ts;\nvar processingTime = Date.now() - metadata.arrivalTs;\nmsg = {\n arrivalLatency: arrivalLatency,\n processingTime: processingTime\n};\nreturn {msg: msg, metadata: metadata, msgType: msgType};" + }, + "externalId": null + }, + { + "additionalInfo": { + "description": "", + "layoutX": 1438, + "layoutY": 403 + }, + "type": "org.thingsboard.rule.engine.transform.TbChangeOriginatorNode", + "name": "To latencies asset", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "originatorSource": "ENTITY", + "entityType": "ASSET", + "entityNamePattern": "[Monitoring] Latencies", + "relationsQuery": { + "direction": "FROM", + "maxLevel": 1, + "filters": [ + { + "relationType": "Contains", + "entityTypes": [] + } + ], + "fetchLastLevelOnly": false + } + }, + "externalId": null + }, + { + "additionalInfo": { + "description": null, + "layoutX": 1458, + "layoutY": 505 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", + "name": "Save Timeseries", + "debugMode": true, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "defaultTTL": 0 + }, + "externalId": null + }, + { + "additionalInfo": { + "description": "", + "layoutX": 928, + "layoutY": 266 + }, + "type": "org.thingsboard.rule.engine.filter.TbCheckMessageNode", + "name": "Has testData", + "debugMode": false, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "messageNames": [ + "testData" + ], + "metadataNames": [], + "checkAllKeys": true + }, + "externalId": null + }, + { + "additionalInfo": { + "description": null, + "layoutX": 1203, + "layoutY": 327 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", + "name": "Save Timeseries with TTL", + "debugMode": true, + "singletonMode": false, + "configurationVersion": 0, + "configuration": { + "defaultTTL": 180, + "skipLatestPersistence": null, + "useServerTs": null + }, + "externalId": null + } + ], + "connections": [ + { + "fromIndex": 2, + "toIndex": 1, + "type": "Post attributes" + }, + { + "fromIndex": 2, + "toIndex": 3, + "type": "RPC Request from Device" + }, + { + "fromIndex": 2, + "toIndex": 4, + "type": "Other" + }, + { + "fromIndex": 2, + "toIndex": 5, + "type": "RPC Request to Device" + }, + { + "fromIndex": 2, + "toIndex": 16, + "type": "Post telemetry" + }, + { + "fromIndex": 6, + "toIndex": 2, + "type": "False" + }, + { + "fromIndex": 6, + "toIndex": 7, + "type": "True" + }, + { + "fromIndex": 7, + "toIndex": 2, + "type": "False" + }, + { + "fromIndex": 7, + "toIndex": 8, + "type": "True" + }, + { + "fromIndex": 8, + "toIndex": 2, + "type": "Success" + }, + { + "fromIndex": 9, + "toIndex": 10, + "type": "Success" + }, + { + "fromIndex": 10, + "toIndex": 11, + "type": "True" + }, + { + "fromIndex": 11, + "toIndex": 6, + "type": "True" + }, + { + "fromIndex": 12, + "toIndex": 9, + "type": "Success" + }, + { + "fromIndex": 13, + "toIndex": 14, + "type": "Success" + }, + { + "fromIndex": 14, + "toIndex": 15, + "type": "Success" + }, + { + "fromIndex": 16, + "toIndex": 0, + "type": "False" + }, + { + "fromIndex": 16, + "toIndex": 17, + "type": "True" + }, + { + "fromIndex": 17, + "toIndex": 13, + "type": "Success" + } + ], + "ruleChainConnections": null + } +} \ No newline at end of file From 98a87bfd33aa3261119759be757f8c070cfbf11b Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 27 Nov 2023 14:01:23 +0200 Subject: [PATCH 27/33] Monitoring: support for dynamic change of load-balancers list --- .../monitoring/service/BaseHealthChecker.java | 13 ++- .../service/BaseMonitoringService.java | 83 ++++++++++++++----- 2 files changed, 75 insertions(+), 21 deletions(-) diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java index 697573289d..290fd307c6 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java @@ -15,6 +15,7 @@ */ package org.thingsboard.monitoring.service; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -30,13 +31,17 @@ import org.thingsboard.monitoring.util.TbStopWatch; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; @RequiredArgsConstructor @Slf4j public abstract class BaseHealthChecker { + @Getter protected final C config; + @Getter protected final T target; private Object info; @@ -48,6 +53,9 @@ public abstract class BaseHealthChecker> associates = new HashMap<>(); + public static final String TEST_TELEMETRY_KEY = "testData"; @PostConstruct @@ -84,6 +92,10 @@ public abstract class BaseHealthChecker { + healthChecker.check(wsClient); + }); } private void checkWsUpdate(WsClient wsClient, String testValue) { @@ -99,7 +111,6 @@ public abstract class BaseHealthChecker, T ext tbClient.logIn(); configs.forEach(config -> { config.getTargets().forEach(target -> { - initHealthChecker(target, config); + BaseHealthChecker healthChecker = initHealthChecker(target, config); + healthCheckers.add(healthChecker); + if (target.isCheckDomainIps()) { - initIpsHealthCheckers(target, config); + getAssociatedUrls(target.getBaseUrl()).forEach(url -> { + healthChecker.getAssociates().put(url, initHealthChecker(createTarget(url), config)); + }); } }); }); } - private void initHealthChecker(T target, C config) { + private BaseHealthChecker initHealthChecker(T target, C config) { BaseHealthChecker healthChecker = (BaseHealthChecker) createHealthChecker(config, target); log.info("Initializing {} for {}", healthChecker.getClass().getSimpleName(), target.getBaseUrl()); healthChecker.initialize(tbClient); devices.add(target.getDeviceId()); - healthCheckers.add(healthChecker); - } - - @SneakyThrows - private void initIpsHealthCheckers(T target, C config) { - URI baseUrl = new URI(target.getBaseUrl()); - String domain = baseUrl.getHost(); - - Set ips = Arrays.stream(InetAddress.getAllByName(domain)) - .map(InetAddress::getHostAddress) - .collect(Collectors.toSet()); - for (String ip : ips) { - String url = new URI(baseUrl.getScheme(), null, ip, baseUrl.getPort(), "", null, null).toString(); - initHealthChecker(createTarget(url), config); - } + return healthChecker; } public final void runChecks() { @@ -108,9 +101,8 @@ public abstract class BaseMonitoringService, T ext try (WsClient wsClient = wsClientFactory.createClient(accessToken)) { wsClient.subscribeForTelemetry(devices, TransportHealthChecker.TEST_TELEMETRY_KEY).waitForReply(); - for (BaseHealthChecker healthChecker : healthCheckers) { - healthChecker.check(wsClient); + check(healthChecker, wsClient); } } reporter.reportLatencies(tbClient); @@ -124,6 +116,57 @@ public abstract class BaseMonitoringService, T ext } } + private void check(BaseHealthChecker healthChecker, WsClient wsClient) throws Exception { + healthChecker.check(wsClient); + + T target = healthChecker.getTarget(); + if (target.isCheckDomainIps()) { + Set associatedUrls = getAssociatedUrls(target.getBaseUrl()); + Map> associates = healthChecker.getAssociates(); + Set prevAssociatedUrls = new HashSet<>(associates.keySet()); + + boolean changed = false; + for (String url : associatedUrls) { + if (!prevAssociatedUrls.contains(url)) { + BaseHealthChecker associate = initHealthChecker(createTarget(url), healthChecker.getConfig()); + associates.put(url, associate); + changed = true; + } + } + for (String url : prevAssociatedUrls) { + if (!associatedUrls.contains(url)) { + stopHealthChecker(healthChecker); + associates.remove(url); + changed = true; + } + } + if (changed) { + log.info("Updated IPs for {}: {} (old list: {})", target.getBaseUrl(), associatedUrls, prevAssociatedUrls); + } + } + } + + @SneakyThrows + private Set getAssociatedUrls(String baseUrl) { + URI url = new URI(baseUrl); + return Arrays.stream(InetAddress.getAllByName(url.getHost())) + .map(InetAddress::getHostAddress) + .map(ip -> { + try { + return new URI(url.getScheme(), null, ip, url.getPort(), "", null, null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toSet()); + } + + private void stopHealthChecker(BaseHealthChecker healthChecker) throws Exception { + healthChecker.destroyClient(); + devices.remove(healthChecker.getTarget().getDeviceId()); + log.info("Stopped {} for {}", healthChecker.getClass().getSimpleName(), healthChecker.getTarget().getBaseUrl()); + } + protected abstract BaseHealthChecker createHealthChecker(C config, T target); protected abstract T createTarget(String baseUrl); From ee8dd9f98767d4f4b575031ebe5a089a76a30e79 Mon Sep 17 00:00:00 2001 From: nick Date: Mon, 27 Nov 2023 18:23:05 +0200 Subject: [PATCH 28/33] stringToBytes: comments2 --- .../script/api/tbel/TbUtilsTest.java | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index 3a3fe22000..7d5ae00dcf 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -423,13 +423,33 @@ public class TbUtilsTest { Assert.assertTrue(e.getMessage().contains("Failed radix: [10] for value: \"1F\"!")); } + List listIntString = new ArrayList<>(); + listIntString.add("-129"); + try { + TbUtils.bytesToString(listIntString); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("The value '-129' could not be correctly converted to a byte. " + + "Integer to byte conversion requires the use of only 8 bits (with a range of min/max = -128/255)!")); + } + + listIntString.add(0, "256"); + try { + TbUtils.bytesToString(listIntString); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("The value '256' could not be correctly converted to a byte. " + + "Integer to byte conversion requires the use of only 8 bits (with a range of min/max = -128/255)!")); + } + ArrayList listIntBytes = new ArrayList<>(); listIntBytes.add(-129); try { TbUtils.bytesToString(listIntBytes); Assert.fail("Should throw NumberFormatException"); } catch (NumberFormatException e) { - Assert.assertTrue(e.getMessage().contains("The value -129 could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255")); + Assert.assertTrue(e.getMessage().contains("The value '-129' could not be correctly converted to a byte. " + + "Integer to byte conversion requires the use of only 8 bits (with a range of min/max = -128/255)!")); } listIntBytes.add(0, 256); @@ -437,7 +457,20 @@ public class TbUtilsTest { TbUtils.bytesToString(listIntBytes); Assert.fail("Should throw NumberFormatException"); } catch (NumberFormatException e) { - Assert.assertTrue(e.getMessage().contains("The value 256 could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255")); + Assert.assertTrue(e.getMessage().contains("The value '256' could not be correctly converted to a byte. " + + "Integer to byte conversion requires the use of only 8 bits (with a range of min/max = -128/255)!")); + } + + ArrayList listObjects = new ArrayList<>(); + ArrayList listStringObjects = new ArrayList<>(); + listStringObjects.add("0xFD"); + listObjects.add(listStringObjects); + try { + TbUtils.bytesToString(listObjects); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("The value '[0xFD]' could not be correctly converted to a byte. " + + "Must be a HexDecimal/String/Integer/Byte format !")); } } From 3af8b1a33e108f5be735fda54cfa66ec014a5b69 Mon Sep 17 00:00:00 2001 From: nick Date: Mon, 27 Nov 2023 18:24:22 +0200 Subject: [PATCH 29/33] stringToBytes: comments2 --- .../thingsboard/script/api/tbel/TbUtils.java | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 8fde4cd332..4a3f2d306b 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -213,18 +213,16 @@ public class TbUtils { private static byte[] bytesFromList(List bytesList) { byte[] bytes = new byte[bytesList.size()]; for (int i = 0; i < bytesList.size(); i++) { - if (bytesList.get(i) instanceof Integer) { - byte val = ((Integer) bytesList.get(i)).byteValue(); - if (((Integer) bytesList.get(i)).intValue() > 255 || ((Integer) bytesList.get(i)).intValue() < -128){ - throw new NumberFormatException("The value " + bytesList.get(i) + " could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255) !"); - } else { - bytes[i] = val; } - } else if (bytesList.get(i) instanceof String) { - bytes[i] = parseInt((String) bytesList.get(i)).byteValue(); - } else if (bytesList.get(i) instanceof Byte) { - bytes[i] = (byte) bytesList.get(i); + Object objectVal = bytesList.get(i); + if (objectVal instanceof Integer) { + bytes[i] = isValidIntegerToByte((Integer) objectVal); + } else if (objectVal instanceof String) { + bytes[i] = isValidIntegerToByte(parseInt((String) objectVal)); + } else if (objectVal instanceof Byte) { + bytes[i] = (byte) objectVal; } else { - throw new NumberFormatException("Value \"" + bytesList.get(i) + " could not be converted to a byte. Must be format HexDecimal/String/Integer !"); + throw new NumberFormatException("The value '" + objectVal + "' could not be correctly converted to a byte. " + + "Must be a HexDecimal/String/Integer/Byte format !"); } } return bytes; @@ -655,7 +653,7 @@ public class TbUtils { } } - public static boolean isValidRadix(String value, int radix) { + private static boolean isValidRadix(String value, int radix) { for (int i = 0; i < value.length(); i++) { if (i == 0 && value.charAt(i) == '-') { if (value.length() == 1) @@ -669,4 +667,12 @@ public class TbUtils { return true; } + private static byte isValidIntegerToByte (Integer val) { + if (val > 255 || val.intValue() < -128) { + throw new NumberFormatException("The value '" + val + "' could not be correctly converted to a byte. " + + "Integer to byte conversion requires the use of only 8 bits (with a range of min/max = -128/255)!"); + } else { + return val.byteValue(); + } + } } From 8ac80476ad2f3768d4f3ecefadd401da607b70a1 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 27 Nov 2023 23:13:03 +0100 Subject: [PATCH 30/33] added encode/decode time stats and refactoring --- .../cache/RedisTbTransactionalCache.java | 4 ++++ .../server/common/data/FstStatsService.java | 4 ++++ .../queue/util/ProtoWithFSTService.java | 8 +++++++- .../common/stats/FstStatsServiceImpl.java | 20 +++++++++++++++++-- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index c2dd66dde0..01a18ad7ad 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -84,8 +84,10 @@ public abstract class RedisTbTransactionalCache clazz); + void recordEncodeTime(Class clazz, long startTime); + + void recordDecodeTime(Class clazz, long startTime); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java index e35c86e070..204502068c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java @@ -36,8 +36,12 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { @Override public Optional decode(byte[] byteArray) { try { + long startTime = System.nanoTime(); Optional optional = Optional.ofNullable(FSTUtils.decode(byteArray)); - optional.ifPresent(obj -> fstStatsService.incrementDecode(obj.getClass())); + optional.ifPresent(obj -> { + fstStatsService.recordDecodeTime(obj.getClass(), startTime); + fstStatsService.incrementDecode(obj.getClass()); + }); return optional; } catch (IllegalArgumentException e) { log.error("Error during deserialization message, [{}]", e.getMessage()); @@ -48,7 +52,9 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { @Override public byte[] encode(T msq) { + long startTime = System.nanoTime(); var bytes = FSTUtils.encode(msq); + fstStatsService.recordEncodeTime(msq.getClass(), startTime); fstStatsService.incrementEncode(msq.getClass()); return bytes; } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java index 5d78b8c36a..b5cb25f2dc 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java @@ -15,28 +15,44 @@ */ package org.thingsboard.server.common.stats; +import io.micrometer.core.instrument.Timer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.FstStatsService; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; @Service public class FstStatsServiceImpl implements FstStatsService { private final ConcurrentHashMap encodeCounters = new ConcurrentHashMap<>(); private final ConcurrentHashMap decodeCounters = new ConcurrentHashMap<>(); + private final ConcurrentHashMap encodeTimers = new ConcurrentHashMap<>(); + private final ConcurrentHashMap decodeTimer = new ConcurrentHashMap<>(); @Autowired private StatsFactory statsFactory; @Override public void incrementEncode(Class clazz) { - encodeCounters.computeIfAbsent(clazz.getSimpleName(), key -> statsFactory.createStatsCounter("fstEncode", key)).increment(); + encodeCounters.computeIfAbsent(clazz.getSimpleName(), key -> statsFactory.createStatsCounter("fst_encode", key)).increment(); } @Override public void incrementDecode(Class clazz) { - decodeCounters.computeIfAbsent(clazz.getSimpleName(), key -> statsFactory.createStatsCounter("fstDecode", key)).increment(); + decodeCounters.computeIfAbsent(clazz.getSimpleName(), key -> statsFactory.createStatsCounter("fst_decode", key)).increment(); + } + + @Override + public void recordEncodeTime(Class clazz, long startTime) { + encodeTimers.computeIfAbsent(clazz.getSimpleName(), + key -> statsFactory.createTimer("fst_encode_time", "statsName", key)).record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); + } + + @Override + public void recordDecodeTime(Class clazz, long startTime) { + decodeTimer.computeIfAbsent(clazz.getSimpleName(), + key -> statsFactory.createTimer("fst_decode_time", "statsName", key)).record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); } } From 94315607d4c6adb96b3c5917d14874495f4d1ce7 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 28 Nov 2023 13:17:12 +0200 Subject: [PATCH 31/33] Edge Requests Service - fetch only first level of relation from cloud to edge --- .../rpc/sync/DefaultEdgeRequestsService.java | 2 +- .../server/edge/RelationEdgeTest.java | 30 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index af9f72e819..cd23f19556 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -294,7 +294,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { private ListenableFuture> findRelationByQuery(TenantId tenantId, Edge edge, EntityId entityId, EntitySearchDirection direction) { EntityRelationsQuery query = new EntityRelationsQuery(); - query.setParameters(new RelationsSearchParameters(entityId, direction, -1, false)); + query.setParameters(new RelationsSearchParameters(entityId, direction, 1, false)); return relationService.findByQuery(tenantId, query); } diff --git a/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java index 04e6f3a3fb..cf82a85a13 100644 --- a/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java @@ -129,14 +129,24 @@ public class RelationEdgeTest extends AbstractEdgeTest { Device device = findDeviceByName("Edge Device 1"); Asset asset = findAssetByName("Edge Asset 1"); - EntityRelation relation = new EntityRelation(); - relation.setType("test"); - relation.setFrom(device.getId()); - relation.setTo(asset.getId()); - relation.setTypeGroup(RelationTypeGroup.COMMON); + EntityRelation deviceToAssetRelation = new EntityRelation(); + deviceToAssetRelation.setType("test"); + deviceToAssetRelation.setFrom(device.getId()); + deviceToAssetRelation.setTo(asset.getId()); + deviceToAssetRelation.setTypeGroup(RelationTypeGroup.COMMON); edgeImitator.expectMessageAmount(1); - doPost("/api/relation", relation); + doPost("/api/relation", deviceToAssetRelation); + Assert.assertTrue(edgeImitator.waitForMessages()); + + EntityRelation assetToTenantRelation = new EntityRelation(); + assetToTenantRelation.setType("test"); + assetToTenantRelation.setFrom(asset.getId()); + assetToTenantRelation.setTo(tenantId); + assetToTenantRelation.setTypeGroup(RelationTypeGroup.COMMON); + + edgeImitator.expectMessageAmount(1); + doPost("/api/relation", assetToTenantRelation); Assert.assertTrue(edgeImitator.waitForMessages()); UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); @@ -159,16 +169,16 @@ public class RelationEdgeTest extends AbstractEdgeTest { Assert.assertTrue(latestMessage instanceof RelationUpdateMsg); RelationUpdateMsg relationUpdateMsg = (RelationUpdateMsg) latestMessage; Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, relationUpdateMsg.getMsgType()); - Assert.assertEquals(relation.getType(), relationUpdateMsg.getType()); + Assert.assertEquals(deviceToAssetRelation.getType(), relationUpdateMsg.getType()); UUID fromUUID = new UUID(relationUpdateMsg.getFromIdMSB(), relationUpdateMsg.getFromIdLSB()); EntityId fromEntityId = EntityIdFactory.getByTypeAndUuid(relationUpdateMsg.getFromEntityType(), fromUUID); - Assert.assertEquals(relation.getFrom(), fromEntityId); + Assert.assertEquals(deviceToAssetRelation.getFrom(), fromEntityId); UUID toUUID = new UUID(relationUpdateMsg.getToIdMSB(), relationUpdateMsg.getToIdLSB()); EntityId toEntityId = EntityIdFactory.getByTypeAndUuid(relationUpdateMsg.getToEntityType(), toUUID); - Assert.assertEquals(relation.getTo(), toEntityId); + Assert.assertEquals(deviceToAssetRelation.getTo(), toEntityId); - Assert.assertEquals(relation.getTypeGroup().name(), relationUpdateMsg.getTypeGroup()); + Assert.assertEquals(deviceToAssetRelation.getTypeGroup().name(), relationUpdateMsg.getTypeGroup()); } } From 026f1748127db2da4cd944ad9c8b35d607b6b2b6 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 28 Nov 2023 17:29:01 +0100 Subject: [PATCH 32/33] fixed rule-engine stats --- ...tomOAuth2AuthorizationRequestResolver.java | 2 + .../TbRuleEngineSecurityConfiguration.java | 44 +++++++++++++++++++ ...eOAuth2AuthorizationRequestRepository.java | 2 + 3 files changed, 48 insertions(+) create mode 100644 application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java b/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java index 4d227345b1..e0d5d3e04e 100644 --- a/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java +++ b/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java @@ -38,6 +38,7 @@ import org.springframework.web.util.UriComponentsBuilder; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.dao.oauth2.OAuth2Configuration; import org.thingsboard.server.dao.oauth2.OAuth2Service; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.oauth2.TbOAuth2ParameterNames; import org.thingsboard.server.service.security.model.token.OAuth2AppTokenFactory; import org.thingsboard.server.utils.MiscUtils; @@ -51,6 +52,7 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; +@TbCoreComponent @Service @Slf4j public class CustomOAuth2AuthorizationRequestResolver implements OAuth2AuthorizationRequestResolver { diff --git a/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java new file mode 100644 index 0000000000..7b3a752f57 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.security.SecurityProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.annotation.Order; +import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.web.SecurityFilterChain; + +@Configuration +@EnableWebSecurity +@EnableGlobalMethodSecurity(prePostEnabled = true) +@Order(SecurityProperties.BASIC_AUTH_ORDER) +@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") +public class TbRuleEngineSecurityConfiguration { + + @Bean + SecurityFilterChain filterChain(HttpSecurity http) throws Exception { + http.headers().cacheControl().and().frameOptions().disable() + .and().cors().and().csrf().disable() + .authorizeRequests() + .antMatchers("/actuator/prometheus").permitAll() + .anyRequest().authenticated(); + return http.build(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/HttpCookieOAuth2AuthorizationRequestRepository.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/HttpCookieOAuth2AuthorizationRequestRepository.java index d63f759dd3..98ccb2c1be 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/HttpCookieOAuth2AuthorizationRequestRepository.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/HttpCookieOAuth2AuthorizationRequestRepository.java @@ -18,11 +18,13 @@ package org.thingsboard.server.service.security.auth.oauth2; import org.springframework.security.oauth2.client.web.AuthorizationRequestRepository; import org.springframework.security.oauth2.core.endpoint.OAuth2AuthorizationRequest; import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.util.TbCoreComponent; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @Component +@TbCoreComponent public class HttpCookieOAuth2AuthorizationRequestRepository implements AuthorizationRequestRepository { public static final String OAUTH2_AUTHORIZATION_REQUEST_COOKIE_NAME = "oauth2_auth_request"; public static final String PREV_URI_PARAMETER = "prevUri"; From 9b51bf7b997e71f676963e91559512c835022a70 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 30 Nov 2023 16:42:31 +0100 Subject: [PATCH 33/33] fixed flaky test --- .../controller/DeviceControllerTest.java | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index cefe24e935..7cafe2bbcb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -28,12 +28,14 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.mockito.AdditionalAnswers; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; import org.springframework.test.context.ContextConfiguration; +import org.testcontainers.shaded.org.awaitility.Awaitility; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; @@ -72,6 +74,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.state.DeviceStateService; @@ -83,7 +86,6 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -1345,27 +1347,27 @@ public class DeviceControllerTest extends AbstractControllerTest { ActionType.ASSIGNED_TO_TENANT, savedDifferentTenant.getId().getId().toString(), savedDifferentTenant.getTitle()); testNotificationUpdateGatewayNever(); - Mockito.verify(deviceStateService, times(1)).onQueueMsg( - argThat(proto -> - proto.getTenantIdMSB() == savedTenant.getUuidId().getMostSignificantBits() && - proto.getTenantIdLSB() == savedTenant.getUuidId().getLeastSignificantBits() && - proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && - proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && - proto.getDeleted() - ), - any() - ); - - Mockito.verify(deviceStateService, times(1)).onQueueMsg( - argThat(proto -> - proto.getTenantIdMSB() == savedDifferentTenant.getUuidId().getMostSignificantBits() && - proto.getTenantIdLSB() == savedDifferentTenant.getUuidId().getLeastSignificantBits() && - proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && - proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && - proto.getAdded() - ), - any() - ); + ArgumentCaptor protoCaptor = ArgumentCaptor.forClass(TransportProtos.DeviceStateServiceMsgProto.class); + + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> { + Mockito.verify(deviceStateService, Mockito.atLeastOnce()).onQueueMsg(protoCaptor.capture(), any()); + return protoCaptor.getAllValues().stream().anyMatch(proto -> + proto.getTenantIdMSB() == savedTenant.getUuidId().getMostSignificantBits() && + proto.getTenantIdLSB() == savedTenant.getUuidId().getLeastSignificantBits() && + proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && + proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && + proto.getDeleted()); + }); + + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> { + Mockito.verify(deviceStateService, Mockito.atLeastOnce()).onQueueMsg(protoCaptor.capture(), any()); + return protoCaptor.getAllValues().stream().anyMatch(proto -> + proto.getTenantIdMSB() == savedDifferentTenant.getUuidId().getMostSignificantBits() && + proto.getTenantIdLSB() == savedDifferentTenant.getUuidId().getLeastSignificantBits() && + proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && + proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && + proto.getAdded()); + }); login("tenant9@thingsboard.org", "testPassword1");