diff --git a/application/pom.xml b/application/pom.xml index 70f8a0c0c7..808931f8e5 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -132,6 +132,18 @@ org.springframework.boot spring-boot-starter-websocket + + org.springframework.cloud + spring-cloud-starter-oauth2 + + + org.springframework.security + spring-security-oauth2-client + + + org.springframework.security + spring-security-oauth2-jose + io.jsonwebtoken jjwt @@ -181,8 +193,8 @@ logback-classic - javax.mail - mail + com.sun.mail + javax.mail org.apache.curator @@ -220,6 +232,11 @@ tools test + + org.thingsboard + rest-client + test + org.springframework.boot spring-boot-starter-test @@ -547,7 +564,7 @@ - org.fortasoft + org.thingsboard gradle-maven-plugin diff --git a/application/src/main/conf/logback.xml b/application/src/main/conf/logback.xml index 295073bb82..46b42182fb 100644 --- a/application/src/main/conf/logback.xml +++ b/application/src/main/conf/logback.xml @@ -36,6 +36,7 @@ + diff --git a/application/src/main/data/json/demo/dashboards/gateways.json b/application/src/main/data/json/demo/dashboards/gateways.json new file mode 100644 index 0000000000..7d75908229 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/gateways.json @@ -0,0 +1,1274 @@ +{ + "title": "Gateways", + "configuration": { + "widgets": { + "94715984-ae74-76e4-20b7-2f956b01ed80": { + "isSystemType": true, + "bundleAlias": "entity_admin_widgets", + "typeAlias": "device_admin_table", + "type": "latest", + "title": "New widget", + "sizeX": 24, + "sizeY": 12, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSearch": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "entityName", + "displayEntityName": true, + "displayEntityType": false, + "entitiesTitle": "List of gateways", + "enableSelectColumnDisplay": true, + "displayEntityLabel": false, + "entityNameColumnTitle": "Gateway Name" + }, + "title": "Devices gateway table", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "active", + "type": "attribute", + "label": "Active", + "color": "#2196f3", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": true, + "useCellContentFunction": true, + "cellContentFunction": "value = '⬤';\nreturn value;", + "cellStyleFunction": "var color;\nif (value == 'false') {\n color = '#EB5757';\n} else {\n color = '#27AE60';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" + }, + "_hash": 0.3646047595211721 + }, + { + "name": "eventsSent", + "type": "timeseries", + "label": "Sent", + "color": "#4caf50", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.7235710720767985 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "Events", + "color": "#f44336", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.5085933386303254 + }, + { + "name": "LOGS", + "type": "timeseries", + "label": "Latest log", + "color": "#ffc107", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.3504240371585048, + "postFuncBody": "if(value) {\n return value.substring(0, 31) + \"...\";\n} else {\n return '';\n}" + }, + { + "name": "RemoteLoggingLevel", + "type": "attribute", + "label": "Log level", + "color": "#607d8b", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.9785994222542516 + } + ], + "entityAliasId": "3e0f533a-0db1-3292-184f-06e73535061a" + } + ], + "showTitleIcon": true, + "titleIcon": "list", + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "List device", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": { + "headerButton": [ + { + "id": "70837a9d-c3de-a9a7-03c5-dccd14998758", + "name": "Add device", + "icon": "add", + "type": "customPretty", + "customHtml": "\n \n \n \n Add device\n \n \n \n \n \n \n \n \n \n \n \n \n Device name\n \n \n Device name is required.\n \n \n \n \n Latitude\n \n \n \n Longitude\n \n \n \n \n Label\n \n \n \n \n \n \n Create\n Cancel\n \n \n\n", + "customCss": "", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddDeviceDialog();\n\nfunction openAddDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope', '$mdDialog',\n AddDeviceDialogController\n ],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddDeviceDialogController($scope, $mdDialog) {\n let vm = this;\n vm.types = types;\n vm.attributes = {};\n vm.deviceType = \"gateway\";\n\n vm.cancel = () => {\n $mdDialog.hide();\n };\n\n vm.save = () => {\n vm.loading = true;\n $scope.addDeviceForm.$setPristine();\n let device = {\n additionalInfo: {gateway: true},\n name: vm.deviceName,\n type: vm.deviceType,\n label: vm.deviceLabel\n };\n deviceService.saveDevice(device).then(\n (device) => {\n saveAttributes(device.id).then(\n () => {\n vm.loading = false;\n updateAliasData();\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n };\n\n function saveAttributes(entityId) {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({\n key: key,\n value: vm.attributes[key]\n });\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(\n entityId.entityType, entityId.id,\n \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n\n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController\n .resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController\n .setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController\n .getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast(\n 'widgetForceReInit');\n });\n }\n}" + } + ], + "actionCellButton": [ + { + "id": "78845501-234e-a452-6819-82b5b776e99f", + "name": "Configuration", + "icon": "settings", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname__config", + "openRightLayout": false, + "setEntityId": true + }, + { + "id": "f6ffdba8-e40f-2b8d-851b-f5ecaf18606b", + "name": "Graphs", + "icon": "show_chart", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname_grafic", + "setEntityId": true + }, + { + "id": "242671f3-76c6-6982-7acc-6f12addf0ccc", + "name": "Edit device", + "icon": "edit", + "type": "customPretty", + "customHtml": "\n \n \n \n Edit device\n \n \n \n \n \n \n \n \n \n \n \n \n Device name\n \n \n Device name is required.\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n Latitude\n \n \n \n Longitude\n \n \n \n \n Label\n \n \n \n \n \n \n Create\n Cancel\n \n \n", + "customCss": "/*=======================================================================*/\n/*========== There are two examples: for edit and add entity ==========*/\n/*=======================================================================*/\n/*======================== Edit entity example ========================*/\n/*=======================================================================*/\n/*\n.edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n\n.relations-list.old-relations tb-entity-select tb-entity-autocomplete button {\n display: none;\n} \n*/\n/*========================================================================*/\n/*========================= Add entity example =========================*/\n/*========================================================================*/\n/*\n.add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n*/\n", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n \nopenEditDeviceDialog();\n\nfunction openEditDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', EditDeviceDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction EditDeviceDialogController($scope,$mdDialog) {\n let vm = this;\n vm.types = types;\n vm.loading = false;\n vm.attributes = {};\n \n getEntityInfo();\n \n function getEntityInfo() {\n vm.loading = true;\n deviceService.getDevice(entityId.id).then(\n (device) => {\n attributeService.getEntityAttributesValues(entityId.entityType, entityId.id, 'SERVER_SCOPE').then(\n (data) => {\n if (data.length) {\n getEntityAttributes(data);\n }\n vm.device = device;\n vm.loading = false;\n } \n );\n }\n )\n }\n \n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n vm.save = () => {\n vm.loading = true;\n $scope.editDeviceForm.$setPristine();\n deviceService.saveDevice(vm.device).then(\n () => {\n saveAttributes().then(\n () => {\n updateAliasData();\n vm.loading = false;\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n }\n \n function getEntityAttributes(attributes) {\n for (let i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n }\n \n function saveAttributes() {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n \n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n console.log(widgetContext);\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}\n" + }, + { + "id": "862ec2b7-fbcf-376e-f85f-b77c07f36efa", + "name": "Delete device", + "icon": "delete", + "type": "custom", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteDeviceDialog();\n\nfunction openDeleteDeviceDialog() {\n let title = \"Are you sure you want to delete the device \" + entityName + \"?\";\n let content = \"Be careful, after the confirmation, the device and all related data will become unrecoverable!\";\n let confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(() => {\n deleteDevice();\n })\n}\n\nfunction deleteDevice() {\n deviceService.deleteDevice(entityId.id).then(\n () => {\n updateAliasData();\n }\n );\n}\n\nfunction updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}" + } + ], + "rowClick": [ + { + "id": "ad5fc7e1-5e60-e056-6940-a75a383466a1", + "name": "to_entityname__config", + "icon": "more_horiz", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname__config", + "setEntityId": true, + "stateEntityParamName": "" + } + ] + } + }, + "id": "94715984-ae74-76e4-20b7-2f956b01ed80" + }, + "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 14, + "sizeY": 13, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "LOGS", + "type": "timeseries", + "label": "LOGS", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.3496649158709739, + "postFuncBody": "return value.replace(/ - (.*) - \\[/gi, ' - $1 - [');" + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 2592000000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 10 + }, + "title": "Debug events (logs)", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "eadabbc7-519e-76fc-ba10-b3fe8c18da10" + }, + "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "opcuaEventsProduced", + "type": "timeseries", + "label": "opcuaEventsProduced", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.1477920581839779 + }, + { + "name": "opcuaEventsSent", + "type": "timeseries", + "label": "opcuaEventsSent", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.6500957113784758 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 120000 + }, + "aggregation": { + "type": "NONE", + "limit": 25000 + }, + "hideInterval": false, + "hideAggregation": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Real time information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "f928afc4-30d1-8d0c-e3cf-777f9f9d1155" + }, + "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 7, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "eventsSent", + "type": "timeseries", + "label": "Events", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.8156044798125357 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "Produced", + "color": "#4caf50", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.6538259344015449 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 604800000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 6, + "hideEmptyLines": true + }, + "title": "Total Messages", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": false, + "showMax": false, + "showAvg": true, + "showTotal": false + } + }, + "id": "2a95b473-042d-59d0-2da2-40d0cccb6c8a" + }, + "aaa69366-aacc-9028-65aa-645c0f8533ec": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "eventsSent", + "type": "timeseries", + "label": "eventsSent", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.41414001784591314 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "eventsProduced", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.7819101846284422 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "History information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "aaa69366-aacc-9028-65aa-645c0f8533ec" + }, + "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "bleEventsProduced", + "type": "timeseries", + "label": "bleEventsProduced", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.5625165504526104 + }, + { + "name": "bleEventsSent", + "type": "timeseries", + "label": "bleEventsSent", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.6817950080745288 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 5000, + "timewindowMs": 120000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Real time information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "ce5c7d01-a3ef-5cf0-4578-8505135c23a0" + }, + "466f046d-6005-a168-b107-60fcb2469cd5": { + "isSystemType": true, + "bundleAlias": "gateway_widgets", + "typeAlias": "attributes_card", + "type": "latest", + "title": "New widget", + "sizeX": 7, + "sizeY": 5, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "eventsTitle": "Gateway Events Form", + "eventsReg": [ + "EventsProduced", + "EventsSent" + ] + }, + "title": "Gateway events", + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "dropShadow": true, + "enableFullscreen": true, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": false, + "actions": {} + }, + "id": "466f046d-6005-a168-b107-60fcb2469cd5" + }, + "8fc32225-164f-3258-73f7-e6b6d959cf0b": { + "isSystemType": true, + "bundleAlias": "gateway_widgets", + "typeAlias": "config_form_latest", + "type": "latest", + "title": "New widget", + "sizeX": 10, + "sizeY": 9, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "gatewayTitle": "Gateway configuration (Single device)", + "readOnly": false + }, + "title": "New Gateway configuration (Single device)", + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "dropShadow": true, + "enableFullscreen": true, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": false, + "actions": {} + }, + "id": "8fc32225-164f-3258-73f7-e6b6d959cf0b" + }, + "063fc179-c9fd-f952-e714-f24e9c43c05c": { + "isSystemType": true, + "bundleAlias": "control_widgets", + "typeAlias": "rpcbutton", + "type": "rpc", + "title": "New widget", + "sizeX": 4, + "sizeY": 2, + "config": { + "targetDeviceAliases": [], + "showTitle": false, + "backgroundColor": "#e6e7e8", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "0px", + "settings": { + "requestTimeout": 5000, + "oneWayElseTwoWay": true, + "styleButton": { + "isRaised": true, + "isPrimary": false + }, + "methodParams": "{}", + "methodName": "gateway_reboot", + "buttonText": "GATEWAY REBOOT" + }, + "title": "New RPC Button", + "dropShadow": true, + "enableFullscreen": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "datasources": [], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "targetDeviceAliasIds": [ + "b2487e75-2fa4-f211-142c-434dfd50c70c" + ] + }, + "id": "063fc179-c9fd-f952-e714-f24e9c43c05c" + }, + "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { + "isSystemType": true, + "bundleAlias": "control_widgets", + "typeAlias": "rpcbutton", + "type": "rpc", + "title": "New widget", + "sizeX": 4, + "sizeY": 2, + "config": { + "targetDeviceAliases": [], + "showTitle": false, + "backgroundColor": "#e6e7e8", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "0px", + "settings": { + "requestTimeout": 5000, + "oneWayElseTwoWay": true, + "styleButton": { + "isRaised": true, + "isPrimary": false + }, + "methodName": "gateway_restart", + "methodParams": "{}", + "buttonText": "gateway restart" + }, + "title": "New RPC Button", + "dropShadow": true, + "enableFullscreen": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "datasources": [], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "targetDeviceAliasIds": [ + "b2487e75-2fa4-f211-142c-434dfd50c70c" + ] + }, + "id": "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7" + }, + "6770b6ba-eff8-df05-75f8-c1f9326d4842": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.9743324774725604 + }, + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.5530093635101525 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}Latitude: ${latitude:7}Longitude: ${longitude:7}Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [], + "useMarkerImageFunction": false, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.Mapnik", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 5 + }, + "title": "Gateway Location", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "6770b6ba-eff8-df05-75f8-c1f9326d4842" + } + }, + "states": { + "main_gateway": { + "name": "Gateways", + "root": true, + "layouts": { + "main": { + "widgets": { + "94715984-ae74-76e4-20b7-2f956b01ed80": { + "sizeX": 24, + "sizeY": 12, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "__entityname__config": { + "name": "${entityName} Configuration", + "root": false, + "layouts": { + "main": { + "widgets": { + "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { + "sizeX": 14, + "sizeY": 13, + "row": 0, + "col": 10 + }, + "8fc32225-164f-3258-73f7-e6b6d959cf0b": { + "sizeX": 10, + "sizeY": 9, + "row": 0, + "col": 0 + }, + "063fc179-c9fd-f952-e714-f24e9c43c05c": { + "sizeX": 4, + "sizeY": 2, + "row": 9, + "col": 0 + }, + "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { + "sizeX": 4, + "sizeY": 2, + "row": 11, + "col": 0 + }, + "6770b6ba-eff8-df05-75f8-c1f9326d4842": { + "sizeX": 6, + "sizeY": 4, + "row": 9, + "col": 4 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "__entityname_grafic": { + "name": "${entityName} Details", + "root": false, + "layouts": { + "main": { + "widgets": { + "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 4, + "col": 7 + }, + "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { + "sizeX": 7, + "sizeY": 7, + "row": 5, + "col": 0 + }, + "aaa69366-aacc-9028-65aa-645c0f8533ec": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 0, + "col": 7 + }, + "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 8, + "col": 7 + }, + "466f046d-6005-a168-b107-60fcb2469cd5": { + "sizeX": 7, + "sizeY": 5, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "auto 100%", + "autoFillHeight": true, + "mobileAutoFillHeight": true, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "3e0f533a-0db1-3292-184f-06e73535061a": { + "id": "3e0f533a-0db1-3292-184f-06e73535061a", + "alias": "Gateways", + "filter": { + "type": "deviceType", + "resolveMultiple": true, + "deviceType": "gateway", + "deviceNameFilter": "" + } + }, + "b2487e75-2fa4-f211-142c-434dfd50c70c": { + "id": "b2487e75-2fa4-f211-142c-434dfd50c70c", + "alias": "Current Gateway", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": "", + "defaultStateEntity": null + } + } + }, + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 25000 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "settings": { + "stateControllerId": "entity", + "showTitle": true, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true, + "titleColor": "rgba(0,0,0,0.870588)" + } + }, + "name": "Gateways" +} diff --git a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json new file mode 100644 index 0000000000..b8231ab880 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json @@ -0,0 +1,520 @@ +{ + "title": "Rule Engine Statistics", + "configuration": { + "widgets": { + "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 12, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "successfulMsgs", + "type": "timeseries", + "label": "${entityName} Successful", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.15490750967648736 + }, + { + "name": "failedMsgs", + "type": "timeseries", + "label": "${entityName} Permanent Failures", + "color": "#ef5350", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.4186621166514697 + }, + { + "name": "tmpFailed", + "type": "timeseries", + "label": "${entityName} Processing Failures", + "color": "#ffc107", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.49891007198715376 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 300000 + }, + "aggregation": { + "type": "NONE", + "limit": 8640 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Queue Stats", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "actions": {}, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": false, + "showTotal": true + } + }, + "id": "81987f19-3eac-e4ce-b790-d96e9b54d9a0" + }, + "5eb79712-5c24-3060-7e4f-6af36b8f842d": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 24, + "sizeY": 5, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Rule Chain", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).ruleChainName;" + }, + "_hash": 0.9954481282345906 + }, + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Rule Node", + "color": "#4caf50", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).ruleNodeName;" + }, + "_hash": 0.18580357036589978 + }, + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Latest Error", + "color": "#f44336", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).message;" + }, + "_hash": 0.7255162989552142 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 10 + }, + "title": "Exceptions", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "5eb79712-5c24-3060-7e4f-6af36b8f842d" + }, + "ad3f1417-87a8-750e-fc67-49a2de1466d4": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 12, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "timeoutMsgs", + "type": "timeseries", + "label": "${entityName} Permanent Timeouts", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.565222981550328 + }, + { + "name": "tmpTimeout", + "type": "timeseries", + "label": "${entityName} Processing Timeouts", + "color": "#9c27b0", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.2679547062508352 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 300000 + }, + "aggregation": { + "type": "NONE", + "limit": 8640 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Processing Failures and Timeouts", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "actions": {}, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": false, + "showTotal": true + } + }, + "id": "ad3f1417-87a8-750e-fc67-49a2de1466d4" + } + }, + "states": { + "default": { + "name": "Rule Engine Statistics", + "root": true, + "layouts": { + "main": { + "widgets": { + "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { + "sizeX": 12, + "sizeY": 7, + "mobileHeight": null, + "row": 0, + "col": 0 + }, + "5eb79712-5c24-3060-7e4f-6af36b8f842d": { + "sizeX": 24, + "sizeY": 5, + "row": 7, + "col": 0 + }, + "ad3f1417-87a8-750e-fc67-49a2de1466d4": { + "sizeX": 12, + "sizeY": 7, + "mobileHeight": null, + "row": 0, + "col": 12 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "140f23dd-e3a0-ed98-6189-03c49d2d8018": { + "id": "140f23dd-e3a0-ed98-6189-03c49d2d8018", + "alias": "TbServiceQueues", + "filter": { + "type": "assetType", + "resolveMultiple": true, + "assetType": "TbServiceQueue", + "assetNameFilter": "" + } + } + }, + "timewindow": { + "displayValue": "", + "selectedTab": 0, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false, + "realtime": { + "interval": 1000, + "timewindowMs": 60000 + }, + "history": { + "historyType": 0, + "interval": 1000, + "timewindowMs": 60000, + "fixedTimewindow": { + "startTimeMs": 1586176634823, + "endTimeMs": 1586263034823 + } + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "settings": { + "stateControllerId": "entity", + "showTitle": false, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true + } + }, + "name": "Rule Engine Statistics" +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/dashboards/thermostats.json b/application/src/main/data/json/demo/dashboards/thermostats.json new file mode 100644 index 0000000000..f91d987447 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/thermostats.json @@ -0,0 +1,1227 @@ +{ + "title": "Thermostats", + "configuration": { + "widgets": { + "f33c746c-0dfc-c212-395b-b448c8a17209": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "entities_table", + "type": "latest", + "title": "New widget", + "sizeX": 11, + "sizeY": 11, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSearch": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "entityName", + "displayEntityName": true, + "displayEntityType": false, + "enableSelectColumnDisplay": false, + "entitiesTitle": "Thermostats", + "displayEntityLabel": false, + "entityNameColumnTitle": "Thermostat name" + }, + "title": "Thermostats", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "active", + "type": "attribute", + "label": "Active", + "color": "#2196f3", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": true, + "useCellContentFunction": true, + "cellContentFunction": "value = '⬤';\nreturn value;", + "cellStyleFunction": "var color;\nif (value === \"true\") {\n color = 'rgb(39, 134, 34)';\n} else {\n color = 'rgb(255, 0, 0)';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" + }, + "_hash": 0.9264526512320641 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "Temperature", + "color": "#4caf50", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.9801965063904188, + "units": "°C", + "decimals": 1 + }, + { + "name": "humidity", + "type": "timeseries", + "label": "Humidity", + "color": "#f44336", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.5726727868178358, + "units": "%", + "decimals": 0 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": { + "headerButton": [ + { + "id": "85b803db-90f2-5c63-1388-a378e0eb10d6", + "name": "Edit location", + "icon": "map", + "type": "openDashboardState", + "targetDashboardStateId": "map", + "setEntityId": false + }, + { + "id": "8ab5a518-67d2-b6a2-956d-81fd512294b2", + "name": "Add", + "icon": "add", + "type": "customPretty", + "customHtml": "\n \n \n \n Add thermostat\n \n \n \n \n \n \n \n \n \n Thermostat name\n \n \n Thermostat name is required.\n \n \n \n High temperature alarm\n \n \n High temperature threshold, °C\n \n \n High temperature threshold is required.\n \n \n \n Low humidity alarm\n \n \n Low humidity threshold, %\n \n \n Low humidity threshold is required.\n \n \n \n \n \n Create\n Cancel\n \n \n", + "customCss": ".add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddEntityDialog();\n\nfunction openAddEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', AddEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddEntityDialogController($scope, $mdDialog) {\n var vm = this;\n vm.attributes = {};\n\n vm.save = function() {\n $scope.addEntityForm.$setPristine();\n saveEntityPromise().then(\n function (entity) {\n saveAttributes(entity.id).then(() => {\n updateAliasData();\n $mdDialog.hide();\n });\n }\n );\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n \n function saveEntityPromise() {\n var entity = {\n name: vm.entityName,\n type: \"thermostat\"\n };\n return deviceService.saveDevice(entity);\n }\n \n function saveAttributes(entityId) {\n var attributesArray = [];\n for (var key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when(null);\n } \n }\n \n function updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}" + } + ], + "actionCellButton": [ + { + "id": "ca241cd8-788d-5508-a9ce-74b03ef42a7f", + "name": "Chart", + "icon": "show_chart", + "type": "openDashboardState", + "targetDashboardStateId": "chart", + "setEntityId": true + }, + { + "id": "7506576f-87ba-d3a0-88fb-e304d451776d", + "name": "Edit", + "icon": "edit", + "type": "customPretty", + "customHtml": "\n \n \n \n Edit thermostat {{vm.entityName}}\n \n \n \n \n \n \n \n \n \n Thermostat name\n \n \n \n High temperature alarm\n \n \n High temperature threshold, °C\n \n \n High temperature threshold is required.\n \n \n \n Low humidity alarm\n \n \n Low humidity threshold, %\n \n \n Low humidity threshold is required.\n \n \n \n \n \n Save\n Cancel\n \n \n", + "customCss": ".edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n attributeService = $injector.get('attributeService');\n\nopenEditEntityDialog();\n\nfunction openEditEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', EditEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction EditEntityDialogController($scope,$mdDialog) {\n var vm = this;\n vm.entityId = entityId;\n vm.entityName = entityName;\n vm.entityType = entityId.entityType;\n vm.attributes = {};\n vm.serverAttributes = {};\n getEntityInfo();\n \n vm.save = function() {\n saveAttributes();\n $mdDialog.hide();\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n function getEntityAttributes(attributes) {\n for (var i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n vm.serverAttributes = angular.copy(vm.attributes);\n }\n \n function getEntityInfo() {\n attributeService.getEntityAttributesValues(entityId.entityType, entityId.id, 'SERVER_SCOPE').then(\n function(data){\n if (data.length) {\n getEntityAttributes(data);\n }\n });\n }\n \n function saveAttributes() {\n var attributesArray = [];\n for (var key in vm.attributes) {\n if (vm.attributes[key] !== vm.serverAttributes[key]) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n }\n if (attributesArray.length > 0) {\n attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } \n }\n}" + }, + { + "id": "3488848b-e47d-6af6-659f-5d78369ece5e", + "name": "Delete", + "icon": "delete", + "type": "custom", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n deviceService = $injector.get('deviceService')\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteEntityDialog();\n\nfunction openDeleteEntityDialog() {\n var title = 'Delete thermostat \"' + entityName + '\"';\n var content = 'Are you sure you want to delete the thermostat \"' +\n entityName + '\"?';\n var confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(function() {\n deleteEntity();\n })\n}\n\nfunction deleteEntity() {\n deviceService.deleteDevice(entityId.id).then(\n function success() {\n updateAliasData();\n },\n function fail() {\n showErrorDialog();\n }\n );\n}\n\nfunction updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}\n\nfunction showErrorDialog() {\n var title = 'Error';\n var content = 'An error occurred while deleting the thermostat. Please try again.';\n var alert = $mdDialog.alert()\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .parent(angular.element($document[0].body))\n .targetEvent($event)\n .multiple(true)\n .clickOutsideToClose(true)\n .ok('CLOSE');\n $mdDialog.show(alert);\n}" + } + ], + "rowClick": [] + } + }, + "id": "f33c746c-0dfc-c212-395b-b448c8a17209" + }, + "7943196b-eedb-d422-f9c3-b32d379ad172": { + "isSystemType": true, + "bundleAlias": "alarm_widgets", + "typeAlias": "alarms_table", + "type": "alarm", + "title": "New widget", + "sizeX": 13, + "sizeY": 5, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSelection": true, + "enableSearch": true, + "displayDetails": true, + "allowAcknowledgment": true, + "allowClear": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "-createdTime", + "enableSelectColumnDisplay": false, + "enableStatusFilter": true, + "alarmsTitle": "Alarms" + }, + "title": "New Alarms table", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "alarmSource": { + "type": "entity", + "dataKeys": [ + { + "name": "createdTime", + "type": "alarm", + "label": "Created time", + "color": "#2196f3", + "settings": {}, + "_hash": 0.7308410188824108 + }, + { + "name": "originator", + "type": "alarm", + "label": "Originator", + "color": "#4caf50", + "settings": {}, + "_hash": 0.056085530105439485 + }, + { + "name": "type", + "type": "alarm", + "label": "Type", + "color": "#f44336", + "settings": {}, + "_hash": 0.10212012352561795 + }, + { + "name": "severity", + "type": "alarm", + "label": "Severity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.1777349980531262 + }, + { + "name": "status", + "type": "alarm", + "label": "Status", + "color": "#607d8b", + "settings": {}, + "_hash": 0.7977920750136249 + } + ], + "entityAliasId": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", + "name": "alarms" + }, + "alarmSearchStatus": "ANY", + "alarmsPollingInterval": 5, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": {}, + "datasources": [], + "alarmsMaxCountLoad": 0, + "alarmsFetchSize": 100 + }, + "id": "7943196b-eedb-d422-f9c3-b32d379ad172" + }, + "14a19183-f0b2-d6be-0f62-9863f0a51111": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 18, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "temperature", + "type": "timeseries", + "label": "Temperature", + "color": "#ef5350", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": true, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.7852346160709658, + "units": "°C", + "decimals": 1 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "interval": 30000, + "timewindowMs": 3600000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + }, + "smoothLines": true + }, + "title": "Temperature", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": false + }, + "actions": {} + }, + "id": "14a19183-f0b2-d6be-0f62-9863f0a51111" + }, + "07f49fd5-a73b-d74c-c220-362c20af81f4": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 18, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "humidity", + "type": "timeseries", + "label": "Humidity", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": true, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.28640715926957183, + "units": "%", + "decimals": 0 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "interval": 30000, + "timewindowMs": 3600000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + }, + "smoothLines": true + }, + "title": "Humidity", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": false + }, + "actions": {} + }, + "id": "07f49fd5-a73b-d74c-c220-362c20af81f4" + }, + "c4631f94-2db3-523b-4d09-2a1a0a75d93f": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "update_multiple_attributes", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "alarmTemperature", + "type": "attribute", + "label": "High temperature alarm", + "color": "#4caf50", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "booleanCheckbox", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1 + }, + "_hash": 0.8725278440159361 + }, + { + "name": "thresholdTemperature", + "type": "attribute", + "label": "High temperature threshold, °C", + "color": "#f44336", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "double", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1, + "disabledOnDataKey": "alarmTemperature" + }, + "_hash": 0.7316078472857874 + }, + { + "name": "alarmHumidity", + "type": "attribute", + "label": "Low humidity alarm", + "color": "#ffc107", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "booleanCheckbox", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1 + }, + "_hash": 0.5339673667431057 + }, + { + "name": "thresholdHumidity", + "type": "attribute", + "label": "Low humidity threshold, %", + "color": "#607d8b", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "double", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1, + "disabledOnDataKey": "alarmHumidity" + }, + "_hash": 0.2687091190358901 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showActionButtons": false, + "showResultMessage": true, + "fieldsAlignment": "column", + "fieldsInRow": 2, + "groupTitle": "${entityName}", + "widgetTitle": "Termostat settings" + }, + "title": "New Update Multiple Attributes", + "dropShadow": true, + "enableFullscreen": false, + "enableDataExport": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "c4631f94-2db3-523b-4d09-2a1a0a75d93f" + }, + "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb": { + "isSystemType": true, + "bundleAlias": "maps_v2", + "typeAlias": "openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 13, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#2196f3", + "settings": {}, + "_hash": 0.1371919646686739, + "decimals": 1, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#4caf50", + "settings": {}, + "_hash": 0.043177186765847475, + "decimals": 0, + "postFuncBody": "return value || \"\";" + }, + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#f44336", + "settings": {}, + "_hash": 0.5548964320315584 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#ffc107", + "settings": {}, + "_hash": 0.1803778014971602 + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.30926987994082844 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}Temperature: ${temperature:1} °CHumidity: ${humidity:0} %Thermostat details", + "markerImageSize": 48, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "colorFunction": "\n", + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "defaultCenterPosition": [ + 0, + 0 + ], + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "useLabelFunction": true, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "defaultZoomLevel": 14, + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;" + }, + "title": "Thermostat maps", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "headerButton": [], + "tooltipAction": [ + { + "id": "bef25673-b37a-8821-bc0f-5d6dd3680f24", + "name": "navigate_to_details", + "icon": "more_horiz", + "type": "openDashboardState", + "targetDashboardStateId": "chart", + "setEntityId": true + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb" + }, + "00fb2742-ba1f-7e43-673f-d6c08b72ed06": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 24, + "sizeY": 12, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.3640193654284214 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.49020393887695923 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#f44336", + "settings": {}, + "_hash": 0.5885892766009955, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.21077893588180707, + "postFuncBody": "return value || \"\";" + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.34722983638504346 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}Temperature: ${temperature:1} °CHumidity: ${humidity:0} %Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 12, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;", + "useLabelFunction": true + }, + "title": "New Markers Placement - OpenStreetMap", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "00fb2742-ba1f-7e43-673f-d6c08b72ed06" + }, + "0a430429-9078-9ae6-2b67-e4a15a2bf8bf": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.3640193654284214 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.49020393887695923 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#f44336", + "settings": {}, + "_hash": 0.5885892766009955, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.21077893588180707, + "postFuncBody": "return value || \"\";" + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.34722983638504346 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}Temperature: ${temperature:1} °CHumidity: ${humidity:0} %Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 37.7749, + -122.4194 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 5, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;", + "useLabelFunction": true, + "useDefaultCenterPosition": true + }, + "title": "New Markers Placement - OpenStreetMap", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "0a430429-9078-9ae6-2b67-e4a15a2bf8bf" + } + }, + "states": { + "default": { + "name": "Thermostats", + "root": true, + "layouts": { + "main": { + "widgets": { + "f33c746c-0dfc-c212-395b-b448c8a17209": { + "sizeX": 11, + "sizeY": 11, + "row": 0, + "col": 0 + }, + "7943196b-eedb-d422-f9c3-b32d379ad172": { + "sizeX": 13, + "sizeY": 5, + "row": 0, + "col": 11 + }, + "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb": { + "sizeX": 13, + "sizeY": 6, + "row": 5, + "col": 11 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "map": { + "name": "Edit location", + "root": false, + "layouts": { + "main": { + "widgets": { + "00fb2742-ba1f-7e43-673f-d6c08b72ed06": { + "sizeX": 24, + "sizeY": 12, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "chart": { + "name": "${entityName}", + "root": false, + "layouts": { + "main": { + "widgets": { + "14a19183-f0b2-d6be-0f62-9863f0a51111": { + "sizeX": 18, + "sizeY": 6, + "mobileHeight": null, + "row": 0, + "col": 6 + }, + "07f49fd5-a73b-d74c-c220-362c20af81f4": { + "sizeX": 18, + "sizeY": 6, + "mobileHeight": null, + "row": 6, + "col": 6 + }, + "c4631f94-2db3-523b-4d09-2a1a0a75d93f": { + "sizeX": 6, + "sizeY": 6, + "row": 0, + "col": 0 + }, + "0a430429-9078-9ae6-2b67-e4a15a2bf8bf": { + "sizeX": 6, + "sizeY": 6, + "row": 6, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "68a058e1-fdda-8482-715b-3ae4a488568e": { + "id": "68a058e1-fdda-8482-715b-3ae4a488568e", + "alias": "Thermostats", + "filter": { + "type": "deviceType", + "resolveMultiple": true, + "deviceType": "thermostat", + "deviceNameFilter": "" + } + }, + "12ae98c7-1ea2-52cf-64d5-763e9d993547": { + "id": "12ae98c7-1ea2-52cf-64d5-763e9d993547", + "alias": "Thermostat", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": null, + "defaultStateEntity": null + } + }, + "ce27a9d0-93bf-b7a4-054d-d0369a8cf813": { + "id": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", + "alias": "Thermostat-alarm", + "filter": { + "type": "entityName", + "resolveMultiple": false, + "entityType": "ASSET", + "entityNameFilter": "Thermostat Alarms" + } + } + }, + "timewindow": { + "displayValue": "", + "selectedTab": 0, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false, + "realtime": { + "interval": 1000, + "timewindowMs": 60000 + }, + "history": { + "historyType": 0, + "interval": 1000, + "timewindowMs": 60000, + "fixedTimewindow": { + "startTimeMs": 1587473857304, + "endTimeMs": 1587560257304 + } + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "settings": { + "stateControllerId": "entity", + "showTitle": false, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true + } + }, + "name": "Thermostats" +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json new file mode 100644 index 0000000000..9805c6f996 --- /dev/null +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -0,0 +1,188 @@ +{ + "ruleChain": { + "additionalInfo": null, + "name": "Root Rule Chain", + "firstRuleNodeId": null, + "root": true, + "debugMode": false, + "configuration": null + }, + "metadata": { + "firstNodeIndex": 3, + "nodes": [ + { + "additionalInfo": { + "layoutX": 1069, + "layoutY": 267 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", + "name": "Is Thermostat?", + "debugMode": false, + "configuration": { + "jsScript": "return msg.id.entityType === \"DEVICE\" && msg.type === \"thermostat\";" + } + }, + { + "additionalInfo": { + "layoutX": 824, + "layoutY": 156 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", + "name": "Save Timeseries", + "debugMode": false, + "configuration": { + "defaultTTL": 0 + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 52 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", + "name": "Save Client Attributes", + "debugMode": false, + "configuration": { + "scope": "CLIENT_SCOPE" + } + }, + { + "additionalInfo": { + "layoutX": 347, + "layoutY": 149 + }, + "type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode", + "name": "Message Type Switch", + "debugMode": false, + "configuration": { + "version": 0 + } + }, + { + "additionalInfo": { + "layoutX": 839, + "layoutY": 345 + }, + "type": "org.thingsboard.rule.engine.action.TbLogNode", + "name": "Log RPC from Device", + "debugMode": false, + "configuration": { + "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" + } + }, + { + "additionalInfo": { + "layoutX": 832, + "layoutY": 407 + }, + "type": "org.thingsboard.rule.engine.action.TbLogNode", + "name": "Log Other", + "debugMode": false, + "configuration": { + "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 468 + }, + "type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode", + "name": "RPC Call Request", + "debugMode": false, + "configuration": { + "timeoutInSeconds": 60 + } + }, + { + "additionalInfo": { + "layoutX": 1069, + "layoutY": 90 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", + "name": "Is Thermostat?", + "debugMode": false, + "configuration": { + "jsScript": "return metadata[\"deviceType\"] === \"thermostat\";" + } + }, + { + "additionalInfo": { + "layoutX": 1090, + "layoutY": 360 + }, + "type": "org.thingsboard.rule.engine.action.TbCreateRelationNode", + "name": "Relate to Asset", + "debugMode": false, + "configuration": { + "direction": "FROM", + "relationType": "ToAlarmPropagationAsset", + "entityType": "ASSET", + "entityNamePattern": "Thermostat Alarms", + "entityTypePattern": "AlarmPropagationAsset", + "entityCacheExpiration": 300, + "createEntityIfNotExists": true, + "changeOriginatorToRelatedEntity": false, + "removeCurrentRelations": false + } + } + ], + "connections": [ + { + "fromIndex": 0, + "toIndex": 8, + "type": "True" + }, + { + "fromIndex": 1, + "toIndex": 7, + "type": "Success" + }, + { + "fromIndex": 3, + "toIndex": 5, + "type": "Other" + }, + { + "fromIndex": 3, + "toIndex": 2, + "type": "Post attributes" + }, + { + "fromIndex": 3, + "toIndex": 1, + "type": "Post telemetry" + }, + { + "fromIndex": 3, + "toIndex": 4, + "type": "RPC Request from Device" + }, + { + "fromIndex": 3, + "toIndex": 6, + "type": "RPC Request to Device" + }, + { + "fromIndex": 3, + "toIndex": 0, + "type": "Entity Created" + } + ], + "ruleChainConnections": [ + { + "fromIndex": 7, + "targetRuleChainId": { + "entityType": "RULE_CHAIN", + "id": "25e26570-89ed-11ea-a650-cd6e14e633bd" + }, + "additionalInfo": { + "layoutX": 1109, + "layoutY": 182, + "ruleChainNodeId": "rule-chain-node-10" + }, + "type": "True" + } + ] + } +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json new file mode 100644 index 0000000000..d67052cbc5 --- /dev/null +++ b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json @@ -0,0 +1,141 @@ +{ + "ruleChain": { + "additionalInfo": null, + "name": "Thermostat Alarms", + "firstRuleNodeId": null, + "root": false, + "debugMode": false, + "configuration": null + }, + "metadata": { + "firstNodeIndex": 5, + "nodes": [ + { + "additionalInfo": { + "layoutX": 929, + "layoutY": 67 + }, + "type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode", + "name": "Create Temp Alarm", + "debugMode": false, + "configuration": { + "alarmType": "High Temperature", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.temperature;\nreturn details;", + "severity": "MAJOR", + "propagate": true, + "useMessageAlarmData": false, + "relationTypes": [ + "ToAlarmPropagationAsset" + ] + } + }, + { + "additionalInfo": { + "layoutX": 930, + "layoutY": 201 + }, + "type": "org.thingsboard.rule.engine.action.TbClearAlarmNode", + "name": "Clear Temp Alarm", + "debugMode": false, + "configuration": { + "alarmType": "High Temperature", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;" + } + }, + { + "additionalInfo": { + "layoutX": 930, + "layoutY": 131 + }, + "type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode", + "name": "Create Humidity Alarm", + "debugMode": false, + "configuration": { + "alarmType": "Low Humidity", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.humidity;\nreturn details;", + "severity": "MINOR", + "propagate": true, + "useMessageAlarmData": false, + "relationTypes": [ + "ToAlarmPropagationAsset" + ] + } + }, + { + "additionalInfo": { + "layoutX": 929, + "layoutY": 275 + }, + "type": "org.thingsboard.rule.engine.action.TbClearAlarmNode", + "name": "Clear Humidity Alarm", + "debugMode": false, + "configuration": { + "alarmType": "Low Humidity", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;" + } + }, + { + "additionalInfo": { + "layoutX": 586, + "layoutY": 148 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsSwitchNode", + "name": "Check Alarms", + "debugMode": false, + "configuration": { + "jsScript": "var relations = [];\nif(metadata[\"ss_alarmTemperature\"] === \"true\"){\n if(msg.temperature > metadata[\"ss_thresholdTemperature\"]){\n relations.push(\"NewTempAlarm\");\n } else {\n relations.push(\"ClearTempAlarm\");\n }\n}\nif(metadata[\"ss_alarmHumidity\"] === \"true\"){\n if(msg.humidity < metadata[\"ss_thresholdHumidity\"]){\n relations.push(\"NewHumidityAlarm\");\n } else {\n relations.push(\"ClearHumidityAlarm\");\n }\n}\n\nreturn relations;" + } + }, + { + "additionalInfo": { + "layoutX": 321, + "layoutY": 149 + }, + "type": "org.thingsboard.rule.engine.metadata.TbGetAttributesNode", + "name": "Fetch Configuration", + "debugMode": false, + "configuration": { + "clientAttributeNames": [], + "sharedAttributeNames": [], + "serverAttributeNames": [ + "alarmTemperature", + "thresholdTemperature", + "alarmHumidity", + "thresholdHumidity" + ], + "latestTsKeyNames": [], + "tellFailureIfAbsent": false, + "getLatestValueWithTs": false + } + } + ], + "connections": [ + { + "fromIndex": 4, + "toIndex": 0, + "type": "NewTempAlarm" + }, + { + "fromIndex": 4, + "toIndex": 1, + "type": "ClearTempAlarm" + }, + { + "fromIndex": 4, + "toIndex": 2, + "type": "NewHumidityAlarm" + }, + { + "fromIndex": 4, + "toIndex": 3, + "type": "ClearHumidityAlarm" + }, + { + "fromIndex": 5, + "toIndex": 4, + "type": "Success" + } + ], + "ruleChainConnections": null + } +} \ No newline at end of file diff --git a/application/src/main/data/json/system/widget_bundles/entity_admin_widgets.json b/application/src/main/data/json/system/widget_bundles/entity_admin_widgets.json index c0d0fa7cdf..b65445f3ca 100644 --- a/application/src/main/data/json/system/widget_bundles/entity_admin_widgets.json +++ b/application/src/main/data/json/system/widget_bundles/entity_admin_widgets.json @@ -6,7 +6,7 @@ }, "widgetTypes": [ { - "alias": "device_admin_table2", + "alias": "device_admin_table", "name": "Device admin table", "descriptor": { "type": "latest", @@ -22,7 +22,7 @@ } }, { - "alias": "device_admin_table", + "alias": "asset_admin_table", "name": "Asset admin table", "descriptor": { "type": "latest", diff --git a/application/src/main/data/json/system/widget_bundles/gateway_widgets.json b/application/src/main/data/json/system/widget_bundles/gateway_widgets.json index c963835ef9..7d5dd46086 100644 --- a/application/src/main/data/json/system/widget_bundles/gateway_widgets.json +++ b/application/src/main/data/json/system/widget_bundles/gateway_widgets.json @@ -5,6 +5,38 @@ "image": null }, "widgetTypes": [ + { + "alias": "attributes_card", + "name": "Gateway events", + "descriptor": { + "type": "latest", + "sizeX": 7.5, + "sizeY": 8, + "resources": [], + "templateHtml": "", + "templateCss": "#container {\n overflow: auto;\n}\n\n.tbDatasource-container {\n margin: 5px;\n padding: 8px;\n}\n\n.tbDatasource-title {\n font-size: 1.200rem;\n font-weight: 500;\n padding-bottom: 10px;\n}\n\n.tbDatasource-table {\n width: 100%;\n box-shadow: 0 0 10px #ccc;\n border-collapse: collapse;\n white-space: nowrap;\n font-size: 1.000rem;\n color: #757575;\n}\n\n.tbDatasource-table td {\n position: relative;\n border-top: 1px solid rgba(0, 0, 0, 0.12);\n border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n padding: 0px 18px;\n box-sizing: border-box;\n}", + "controllerScript": "let types;\nlet eventsReg = \"eventsReg\";\n\nself.onInit = function() {\n \n self.ctx.datasourceTitleCells = [];\n self.ctx.valueCells = [];\n self.ctx.labelCells = [];\n types = self.ctx.$scope.$injector.get('types');\n \n if (self.ctx.datasources.length && self.ctx.datasources[0].type === types.datasourceType.entity) {\n getDatasourceKeys(self.ctx.datasources[0]);\n } else {\n processDatasources(self.ctx.datasources);\n }\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.valueCells.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData && cellData.data && cellData.data.length > 0) {\n var tvPair = cellData.data[cellData.data.length -\n 1];\n var value = tvPair[1];\n var textValue;\n //toDo -> + IsNumber\n \n if (isNumber(value)) {\n var decimals = self.ctx.decimals;\n var units = self.ctx.units;\n if (cellData.dataKey.decimals || cellData.dataKey.decimals === 0) {\n decimals = cellData.dataKey.decimals;\n }\n if (cellData.dataKey.units) {\n units = cellData.dataKey.units;\n }\n txtValue = self.ctx.utils.formatValue(value, decimals, units, false);\n }\n else {\n txtValue = value;\n }\n self.ctx.valueCells[i].html(txtValue);\n }\n }\n \n function isNumber(n) {\n return !isNaN(parseFloat(n)) && isFinite(n);\n }\n}\n\nself.onResize = function() {\n var datasourceTitleFontSize = self.ctx.height/8;\n if (self.ctx.width/self.ctx.height <= 1.5) {\n datasourceTitleFontSize = self.ctx.width/12;\n }\n datasourceTitleFontSize = Math.min(datasourceTitleFontSize, 20);\n for (var i = 0; i < self.ctx.datasourceTitleCells.length; i++) {\n self.ctx.datasourceTitleCells[i].css('font-size', datasourceTitleFontSize+'px');\n }\n var valueFontSize = self.ctx.height/9;\n var labelFontSize = self.ctx.height/9;\n if (self.ctx.width/self.ctx.height <= 1.5) {\n valueFontSize = self.ctx.width/15;\n labelFontSize = self.ctx.width/15;\n }\n valueFontSize = Math.min(valueFontSize, 18);\n labelFontSize = Math.min(labelFontSize, 18);\n\n for (i = 0; i < self.ctx.valueCells; i++) {\n self.ctx.valueCells[i].css('font-size', valueFontSize+'px');\n self.ctx.valueCells[i].css('height', valueFontSize*2.5+'px');\n self.ctx.valueCells[i].css('padding', '0px ' + valueFontSize + 'px');\n self.ctx.labelCells[i].css('font-size', labelFontSize+'px');\n self.ctx.labelCells[i].css('height', labelFontSize*2.5+'px');\n self.ctx.labelCells[i].css('padding', '0px ' + labelFontSize + 'px');\n } \n}\n\nfunction processDatasources(datasources) {\n var i = 0;\n var tbDatasource = datasources[i];\n var datasourceId = 'tbDatasource' + i;\n self.ctx.$container.append(\n \"\"\n );\n\n var datasourceContainer = $('#' + datasourceId,\n self.ctx.$container);\n\n datasourceContainer.append(\n \"\" +\n tbDatasource.name + \"\"\n );\n \n var datasourceTitleCell = $('.tbDatasource-title', datasourceContainer);\n self.ctx.datasourceTitleCells.push(datasourceTitleCell);\n \n var tableId = 'table' + i;\n datasourceContainer.append(\n \"\"\n );\n var table = $('#' + tableId, self.ctx.$container);\n\n for (var a = 0; a < tbDatasource.dataKeys.length; a++) {\n var dataKey = tbDatasource.dataKeys[a];\n var labelCellId = 'labelCell' + a;\n var cellId = 'cell' + a;\n table.append(\"\" + dataKey.label +\n \"\");\n var labelCell = $('#' + labelCellId, table);\n self.ctx.labelCells.push(labelCell);\n var valueCell = $('#' + cellId, table);\n self.ctx.valueCells.push(valueCell);\n }\n self.onResize();\n}\n\nfunction getDatasourceKeys (datasource) {\n let attributeService = self.ctx.$scope.$injector.get('attributeService');\n if (datasource.entityId && datasource.entityType) {\n attributeService.getEntityKeys(datasource.entityType, datasource.entityId, \"\" , types.dataKeyType.timeseries).then(\n function(data){\n if (data.length) {\n subscribeForKeys (datasource, data);\n }\n });\n }\n}\n\nfunction subscribeForKeys (datasource, data) {\n let eventsRegVals = self.ctx.settings[eventsReg];\n if (eventsRegVals && eventsRegVals.length > 0) {\n var dataKeys = [];\n data.sort();\n data.forEach(dataValue => {eventsRegVals.forEach(event => {\n if (dataValue.toLowerCase().includes(event.toLowerCase())) {\n var dataKey = {\n type: types.dataKeyType.timeseries,\n name: dataValue,\n label: dataValue,\n settings: {},\n _hash: Math.random()\n };\n dataKeys.push(dataKey);\n }\n })});\n\n if (dataKeys.length) {\n updateSubscription (datasource, dataKeys);\n }\n }\n}\n\nfunction updateSubscription (datasource, dataKeys) {\n var datasources = [\n {\n type: types.datasourceType.entity,\n name: datasource.aliasName,\n aliasName: datasource.aliasName,\n entityAliasId: datasource.entityAliasId,\n dataKeys: dataKeys\n }\n ];\n \n var subscriptionOptions = {\n datasources: datasources,\n useDashboardTimewindow: false,\n type: types.widgetType.latest.value,\n callbacks: {\n onDataUpdated: (subscription) => {\n self.ctx.data = subscription.data;\n self.onDataUpdated();\n }\n }\n };\n \n processDatasources(datasources);\n self.ctx.subscriptionApi.createSubscription(subscriptionOptions, true).then(\n (subscription) => {\n self.ctx.defaultSubscription = subscription;\n }\n );\n}\n\nself.onDestroy = function() {\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\t\n dataKeysOptional: true\n };\n}\n\n", + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"GatewayEventsForm\",\n \"properties\": {\n \"eventsTitle\": {\n \"title\": \"Gateway events form title\",\n \"type\": \"string\",\n \"default\": \"Gateway Events Form\"\n },\n \"eventsReg\": {\n \"title\": \"Events filten.\",\n \"type\": \"array\",\n \"items\": {\n \"title\": \"Event key contains\",\n \"type\": \"string\"\n }\n }\n }\n },\n \"form\": [\n \"eventsTitle\",\n \"eventsReg\"\n ]\n}", + "dataKeySettingsSchema": "{}\n", + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Function Math.round\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.826503672916844,\"funcBody\":\"return Math.round(1000*Math.sin(time/5000));\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{\"eventsTitle\":\"Gateway Events Form\",\"eventsReg\":[]},\"title\":\"Gateway events\",\"showTitleIcon\":false,\"titleIcon\":\"more_horiz\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"titleTooltip\":\"\",\"dropShadow\":true,\"enableFullscreen\":true,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"displayTimewindow\":true,\"showLegend\":false,\"actions\":{}}" + } + }, + { + "alias": "config_form_latest", + "name": "Gateway configuration (Single device)", + "descriptor": { + "type": "latest", + "sizeX": 7.5, + "sizeY": 9, + "resources": [], + "templateHtml": "\n", + "templateCss": "#container {\n overflow: auto;\n}\n\n.tbDatasource-container {\n margin: 5px;\n padding: 8px;\n}\n\n.tbDatasource-title {\n font-size: 1.200rem;\n font-weight: 500;\n padding-bottom: 10px;\n}\n\n.tbDatasource-table {\n width: 100%;\n box-shadow: 0 0 10px #ccc;\n border-collapse: collapse;\n white-space: nowrap;\n font-size: 1.000rem;\n color: #757575;\n}\n\n.tbDatasource-table td {\n position: relative;\n border-top: 1px solid rgba(0, 0, 0, 0.12);\n border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n padding: 0px 18px;\n box-sizing: border-box;\n}", + "controllerScript": "self.onInit = function() {\n var scope = self.ctx.$scope;\n var id = self.ctx.$scope.$injector.get('utils').guid();\n scope.formId = \"form-\"+id;\n scope.ctx = self.ctx;\n scope.isStateForm = true;\n}\n\nself.onResize = function() {\n self.ctx.$scope.$broadcast('gateway-form-resize', self.ctx.$scope.formId);\n}\n\n\nself.onDestroy = function() {\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\t\t\t\n dataKeysOptional: true\t\t\n };\n}\n\n", + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"GatewayConfigForm\",\n \"properties\": {\n \"gatewayTitle\": {\n \"title\": \"Gateway form\",\n \"type\": \"string\",\n \"default\": \"Gateway configuration (Single device)\"\n },\n \"readOnly\": {\n \"title\": \"Read Only\",\n \"type\": \"boolean\",\n \"default\": false\n }\n }\n },\n \"form\": [\n \"gatewayTitle\",\n \"readOnly\"\n ]\n}\n", + "dataKeySettingsSchema": "{}\n", + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Gateway configuration (Single device)\"}" + } + }, { "alias": "extension_configuration_widget", "name": "Extensions table", @@ -20,6 +52,22 @@ "dataKeySettingsSchema": "{}\n", "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"4px\",\"settings\":{},\"title\":\"Extensions table\",\"dropShadow\":true,\"enableFullscreen\":true,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"18px\",\"fontWeight\":400,\"padding\":\"5px 10px 5px 10px\"},\"useDashboardTimewindow\":true,\"showLegend\":false,\"actions\":{}}" } + }, + { + "alias": "gateway_configuration", + "name": "Gateway Configuration", + "descriptor": { + "type": "static", + "sizeX": 8, + "sizeY": 6.5, + "resources": [], + "templateHtml": "\n\n", + "templateCss": "", + "controllerScript": "self.onInit = function() {\n var scope = self.ctx.$scope;\n var id = self.ctx.$scope.$injector.get('utils').guid();\n scope.formId = \"form-\"+id;\n scope.ctx = self.ctx;\n}\n\nself.onResize = function() {\n self.ctx.$scope.$broadcast('gateway-form-resize', self.ctx.$scope.formId);\n}\n", + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Settings\",\n \"properties\": {\n \"widgetTitle\": {\n \"title\": \"Widget title\",\n \"type\": \"string\",\n \"default\": \"Gateway Configuration\"\n },\n \"archiveFileName\": {\n \"title\": \"Default archive file name\",\n \"type\": \"string\",\n \"default\": \"gatewayConfiguration\"\n },\n \"gatewayType\": {\n \"title\": \"Device type for new gateway\",\n \"type\": \"string\",\n \"default\": \"Gateway\"\n },\n \"successfulSave\": {\n \"title\": \"Text message about successfully saved gateway configuration\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"gatewayNameExists\": {\n \"title\": \"Text message when device with entered name is already exists\",\n \"type\": \"string\",\n \"default\": \"\"\n }\n },\n \"required\": []\n },\n \"form\": [\n [\n \"widgetTitle\",\n \"archiveFileName\",\n \"gatewayType\"\n ],\n [\n \"successfulSave\",\n \"gatewayNameExists\"\n ]\n ],\n \"groupInfoes\": [{\n \"formIndex\": 0,\n \"GroupTitle\": \"General settings\"\n }, {\n \"formIndex\": 1,\n \"GroupTitle\": \"Messages settings\"\n }]\n}", + "dataKeySettingsSchema": "{}\n", + "defaultConfig": "{\"datasources\":[{\"type\":\"static\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"rgb(255, 255, 255)\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{\"widgetTitle\":\"Gateway Configuration\",\"archiveFileName\":\"configurationGateway\"},\"title\":\"Gateway Configuration\",\"dropShadow\":true,\"showTitleIcon\":false,\"titleIcon\":\"more_horiz\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"titleTooltip\":\"\",\"enableFullscreen\":true,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"displayTimewindow\":true,\"showLegend\":false,\"actions\":{}}" + } } ] -} \ No newline at end of file +} diff --git a/application/src/main/data/json/system/widget_bundles/input_widgets.json b/application/src/main/data/json/system/widget_bundles/input_widgets.json index 2cc297b263..18765a8b54 100644 --- a/application/src/main/data/json/system/widget_bundles/input_widgets.json +++ b/application/src/main/data/json/system/widget_bundles/input_widgets.json @@ -330,13 +330,13 @@ "name": "Web Camera Input", "descriptor": { "type": "latest", - "sizeX": 9.5, - "sizeY": 6.5, + "sizeX": 7.5, + "sizeY": 3, "resources": [], "templateHtml": "\n", - "templateCss": "#container {\n overflow: auto;\n}\n\n.tbDatasource-container {\n margin: 5px;\n padding: 8px;\n}\n\n.tbDatasource-title {\n font-size: 1.200rem;\n font-weight: 500;\n padding-bottom: 10px;\n}\n\n.tbDatasource-table {\n width: 100%;\n box-shadow: 0 0 10px #ccc;\n border-collapse: collapse;\n white-space: nowrap;\n font-size: 1.000rem;\n color: #757575;\n}\n\n.tbDatasource-table td {\n position: relative;\n border-top: 1px solid rgba(0, 0, 0, 0.12);\n border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n padding: 0px 18px;\n box-sizing: border-box;\n}", + "templateCss": "", "controllerScript": "self.onInit = function() {\n var scope = self.ctx.$scope;\n scope.ctx = self.ctx;\n}\n\nself.onDataUpdated = function() {\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1\n }\n}\n\nself.onDestroy = function() {\n}\n", - "settingsSchema": "{}", + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Web Camera\",\n \"properties\": {\n \"widgetTitle\": {\n \"title\": \"Widget title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"imageFormat\": {\n \"title\": \"Image Format\",\n \"type\": \"string\",\n \"default\": \"image/png\"\n },\n \"imageQuality\":{\n \"title\":\"Image quality that use lossy compression such as jpeg and webp\",\n \"type\":\"number\",\n \"default\": 0.92,\n \"min\": 0,\n \"max\": 1\n },\n \"maxWidth\": {\n \"title\": \"The maximal image width\",\n \"type\": \"number\",\n \"default\": 640\n }, \n \"maxHeight\": {\n \"title\": \"The maximal image heigth\",\n \"type\": \"number\",\n \"default\": 480\n }\n },\n \"required\": []\n },\n \"form\": [\n \"widgetTitle\",\n {\n \"key\": \"imageFormat\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"image/jpeg\",\n \"label\": \"JPEG\"\n },\n {\n \"value\": \"image/png\",\n \"label\": \"PNG\"\n },\n {\n \"value\": \"image/webp\",\n \"label\": \"WEBP\"\n }\n ]\n },\n \"imageQuality\",\n \"maxWidth\",\n \"maxHeight\"\n ]\n}", "dataKeySettingsSchema": "{}\n", "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Web Camera Input\",\"showTitleIcon\":false,\"titleIcon\":\"more_horiz\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"titleTooltip\":\"\",\"dropShadow\":true,\"enableFullscreen\":false,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"displayTimewindow\":true,\"showLegend\":false,\"actions\":{}}" } diff --git a/application/src/main/data/json/system/widget_bundles/maps.json b/application/src/main/data/json/system/widget_bundles/maps.json index 7018349a66..82da974cba 100644 --- a/application/src/main/data/json/system/widget_bundles/maps.json +++ b/application/src/main/data/json/system/widget_bundles/maps.json @@ -128,10 +128,10 @@ "templateHtml": "", "templateCss": ".legend {\n font-size: 13px;\n line-height: 10px;\n}\n\n.legend table { \n border-spacing: 0px;\n border-collapse: separate;\n}\n\n.mouse-events .flot-overlay {\n cursor: crosshair; \n}\n\n", "controllerScript": " self.onInit = function() {\n var $scope = self.ctx.$scope;\n $scope.self = self;\n }\n \n \n self.actionSources = function () {\n return {\n 'tooltipAction': {\n name: 'widget-action.tooltip-tag-action',\n multiple: false\n }\n }\n };\n", - "settingsSchema": "{\n \"schema\": {\n \"title\": \"Openstreet Map Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"mapProvider\": {\n \"title\": \"Map provider\",\n \"type\": \"string\",\n \"default\": \"OpenStreetMap.Mapnik\"\n },\n \"normalizationStep\": {\n \"title\": \"Normalization data step (ms)\",\n \"type\": \"number\",\n \"default\": 1000\n },\n \"latKeyName\": {\n \"title\": \"Latitude key name\",\n \"type\": \"string\",\n \"default\": \"latitude\"\n },\n \"lngKeyName\": {\n \"title\": \"Longitude key name\",\n \"type\": \"string\",\n \"default\": \"longitude\"\n },\n \"polKeyName\": {\n \"title\": \"Polygon key name\",\n \"type\": \"string\",\n \"default\": \"coordinates\"\n },\n \"showLabel\": {\n \"title\": \"Show label\",\n \"type\": \"boolean\",\n \"default\": true\n },\n \"label\": {\n \"title\": \"Label (pattern examples: '${entityName}', '${entityName}: (Text ${keyName} units.)' )\",\n \"type\": \"string\",\n \"default\": \"${entityName}\"\n },\n \"useLabelFunction\": {\n \"title\": \"Use label function\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"labelFunction\": {\n \"title\": \"Label function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"showTooltip\": {\n \"title\": \"Show tooltip\",\n \"type\": \"boolean\",\n \"default\": true\n },\n \"tooltipColor\": {\n \"title\": \"Tooltip background color\",\n \"type\": \"string\",\n \"default\": \"#fff\"\n },\n \"tooltipFontColor\": {\n \"title\": \"Tooltip font color\",\n \"type\": \"string\",\n \"default\": \"#000\"\n },\n \"tooltipOpacity\": {\n \"title\": \"Tooltip opacity (0-1)\",\n \"type\": \"number\",\n \"default\": 1\n },\n \"tooltipPattern\": {\n \"title\": \"Tooltip (for ex. 'Text ${keyName} units.' or Link text')\",\n \"type\": \"string\",\n \"default\": \"${entityName}Latitude: ${latitude:7}Longitude: ${longitude:7}\"\n },\n \"useTooltipFunction\": {\n \"title\": \"Use tooltip function\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"tooltipFunction\": {\n \"title\": \"Tooltip function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"color\": {\n \"title\": \"Path color\",\n \"type\": \"string\"\n },\n \"strokeWeight\": {\n \"title\": \"Stroke weight\",\n \"type\": \"number\",\n \"default\": 2\n },\n \"strokeOpacity\": {\n \"title\": \"Stroke opacity\",\n \"type\": \"number\",\n \"default\": 1\n },\n \"useColorFunction\": {\n \"title\": \"Use path color function\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"colorFunction\": {\n \"title\": \"Path color function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"usePolylineDecorator\": {\n \"title\": \"Use path decorator\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"decoratorSymbol\": {\n \"title\": \"Decorator symbol\",\n \"type\": \"string\",\n \"default\": \"arrowHead\"\n },\n \"decoratorSymbolSize\": {\n \"title\": \"Decorator symbol size (px)\",\n \"type\": \"number\",\n \"default\": 10\n },\n \"useDecoratorCustomColor\": {\n \"title\": \"Use path decorator custom color\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"decoratorCustomColor\": {\n \"title\": \"Decorator custom color\",\n \"type\": \"string\",\n \"default\": \"#000\"\n },\n \"decoratorOffset\": {\n \"title\": \"Decorator offset\",\n \"type\": \"string\",\n \"default\": \"20px\"\n },\n \"endDecoratorOffset\": {\n \"title\": \"End decorator offset\",\n \"type\": \"string\",\n \"default\": \"20px\"\n },\n \"decoratorRepeat\": {\n \"title\": \"Decorator repeat\",\n \"type\": \"string\",\n \"default\": \"20px\"\n },\n \"showPolygon\": {\n \"title\": \"Show polygon\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"polygonTooltipPattern\": {\n \"title\": \"Tooltip (for ex. 'Text ${keyName} units.' or Link text')\",\n \"type\": \"string\",\n \"default\": \"${entityName}TimeStamp: ${ts:7}\"\n },\n \"usePolygonTooltipFunction\": {\n \"title\": \"Use polygon tooltip function\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"polygonTooltipFunction\": {\n \"title\": \"Polygon tooltip function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"polygonColor\": {\n \"title\": \"Polygon color\",\n \"type\": \"string\"\n },\n \"polygonOpacity\": {\n \"title\": \"Polygon opacity\",\n \"type\": \"number\",\n \"default\": 0.5\n },\n \"polygonStrokeColor\": {\n \"title\": \"Polygon border color\",\n \"type\": \"string\"\n },\n \"polygonStrokeOpacity\": {\n \"title\": \"Polygon border opacity\",\n \"type\": \"number\",\n \"default\": 1\n },\n \"polygonStrokeWeight\": {\n \"title\": \"Polygon border weight\",\n \"type\": \"number\",\n \"default\": 1\n },\n \"usePolygonColorFunction\": {\n \"title\": \"Use polygon color function\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"polygonColorFunction\": {\n \"title\": \"Polygon Color function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"showPoints\": {\n \"title\": \"Show points\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"pointColor\": {\n \"title\": \"Point color\",\n \"type\": \"string\"\n },\n \"pointSize\": {\n \"title\": \"Point size (px)\",\n \"type\": \"number\",\n \"default\": 10\n },\n \"usePointAsAnchor\": {\n \"title\": \"Use point as anchor\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"pointAsAnchorFunction\": {\n \"title\": \"Point as anchor function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"pointTooltipOnRightPanel\": {\n \"title\": \"Independant point tooltip\",\n \"type\": \"boolean\",\n \"default\": true\n },\n \"autocloseTooltip\": {\n \"title\": \"Auto-close point popup\",\n \"type\": \"boolean\",\n \"default\": true\n },\n \"defaultMarkerColor\": {\n \"title\": \"color for default marker\",\n \"type\": \"string\"\n },\n \"markerImage\": {\n \"title\": \"Custom marker image\",\n \"type\": \"string\"\n },\n \"markerImageSize\": {\n \"title\": \"Custom marker image size (px)\",\n \"type\": \"number\",\n \"default\": 34\n },\n \"rotationAngle\": {\n \"title\": \"Set additional rotation angle for marker (deg)\",\n \"type\": \"number\",\n \"default\": 180\n },\n \"useMarkerImageFunction\":{\n \"title\":\"Use marker image function\",\n \"type\":\"boolean\",\n \"default\":false\n },\n \"markerImageFunction\":{\n \"title\":\"Marker image function: f(data, images, dsData, dsIndex)\",\n \"type\":\"string\"\n },\n \"markerImages\":{\n \"title\":\"Marker images\",\n \"type\":\"array\",\n \"items\":{\n \"title\":\"Marker image\",\n \"type\":\"string\"\n }\n }\n },\n \"required\": []\n },\n \"form\": [{\n \"key\": \"mapProvider\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [{\n \"value\": \"OpenStreetMap.Mapnik\",\n \"label\": \"OpenStreetMap.Mapnik (Default)\"\n }, {\n \"value\": \"OpenStreetMap.BlackAndWhite\",\n \"label\": \"OpenStreetMap.BlackAndWhite\"\n }, {\n \"value\": \"OpenStreetMap.HOT\",\n \"label\": \"OpenStreetMap.HOT\"\n }, {\n \"value\": \"Esri.WorldStreetMap\",\n \"label\": \"Esri.WorldStreetMap\"\n }, {\n \"value\": \"Esri.WorldTopoMap\",\n \"label\": \"Esri.WorldTopoMap\"\n }, {\n \"value\": \"CartoDB.Positron\",\n \"label\": \"CartoDB.Positron\"\n }, {\n \"value\": \"CartoDB.DarkMatter\",\n \"label\": \"CartoDB.DarkMatter\"\n }]\n }, \"normalizationStep\", \"latKeyName\", \"lngKeyName\", \"polKeyName\", \"showLabel\", \"label\", \"useLabelFunction\", {\n \"key\": \"labelFunction\",\n \"type\": \"javascript\"\n }, \"showTooltip\", {\n \"key\": \"tooltipColor\",\n \"type\": \"color\"\n }, {\n \"key\": \"tooltipFontColor\",\n \"type\": \"color\"\n },\"tooltipOpacity\", {\n \"key\": \"tooltipPattern\",\n \"type\": \"textarea\"\n }, \"useTooltipFunction\", {\n \"key\": \"tooltipFunction\",\n \"type\": \"javascript\"\n }, {\n \"key\": \"color\",\n \"type\": \"color\"\n }, \"useColorFunction\", {\n \"key\": \"colorFunction\",\n \"type\": \"javascript\"\n }, \"usePolylineDecorator\", {\n \"key\": \"decoratorSymbol\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [{\n \"value\": \"arrowHead\",\n \"label\": \"Arrow\"\n }, {\n \"value\": \"dash\",\n \"label\": \"Dash\"\n }]\n }, \"decoratorSymbolSize\", \"useDecoratorCustomColor\", {\n \"key\": \"decoratorCustomColor\",\n \"type\": \"color\"\n }, {\n \"key\": \"decoratorOffset\",\n \"type\": \"textarea\"\n },{\n \"key\": \"endDecoratorOffset\",\n \"type\": \"textarea\"\n }, {\n \"key\": \"decoratorRepeat\",\n \"type\": \"textarea\"\n }, \"strokeWeight\", \"strokeOpacity\", \"showPolygon\", {\n \"key\": \"polygonTooltipPattern\",\n \"type\": \"textarea\"\n },\"usePolygonTooltipFunction\", {\n \"key\": \"polygonTooltipFunction\",\n \"type\": \"javascript\"\n },{\n \"key\": \"polygonColor\",\n \"type\": \"color\"\n },\t\"polygonOpacity\", {\n \"key\": \"polygonStrokeColor\",\n \"type\": \"color\"\n },\t\"polygonStrokeOpacity\",\"polygonStrokeWeight\",\"usePolygonColorFunction\",\t{\n \"key\": \"polygonColorFunction\",\n \"type\": \"javascript\"\n },\"showPoints\",{\n \"key\": \"pointColor\",\n \"type\": \"color\"\n }, \"pointSize\",\"usePointAsAnchor\", {\n \"key\": \"pointAsAnchorFunction\",\n \"type\": \"javascript\"\n },\"pointTooltipOnRightPanel\", \"autocloseTooltip\", {\n \"key\": \"defaultMarkerColor\",\n \"type\": \"color\"\n }, {\n \"key\": \"markerImage\",\n \"type\": \"image\"\n }, \"markerImageSize\", \"rotationAngle\",\"useMarkerImageFunction\",\n {\n \"key\":\"markerImageFunction\",\n \"type\":\"javascript\"\n }, {\n \"key\":\"markerImages\",\n \"items\":[\n {\n \"key\":\"markerImages[]\",\n \"type\":\"image\"\n }\n ]\n }]\n}", + "settingsSchema": "{\n \"schema\": {\n \"title\": \"Openstreet Map Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"mapProvider\": {\n \"title\": \"Map provider\",\n \"type\": \"string\",\n \"default\": \"OpenStreetMap.Mapnik\"\n },\n \"normalizationStep\": {\n \"title\": \"Normalization data step (ms)\",\n \"type\": \"number\",\n \"default\": 1000\n },\n \"latKeyName\": {\n \"title\": \"Latitude key name\",\n \"type\": \"string\",\n \"default\": \"latitude\"\n },\n \"lngKeyName\": {\n \"title\": \"Longitude key name\",\n \"type\": \"string\",\n \"default\": \"longitude\"\n },\n \"polKeyName\": {\n \"title\": \"Polygon key name\",\n \"type\": \"string\",\n \"default\": \"coordinates\"\n },\n \"showLabel\": {\n \"title\": \"Show label\",\n \"type\": \"boolean\",\n \"default\": true\n },\n \"label\": {\n \"title\": \"Label (pattern examples: '${entityName}', '${entityName}: (Text ${keyName} units.)' )\",\n \"type\": \"string\",\n \"default\": \"${entityName}\"\n },\n \"useLabelFunction\": {\n \"title\": \"Use label function\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"labelFunction\": {\n \"title\": \"Label function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"showTooltip\": {\n \"title\": \"Show tooltip\",\n \"type\": \"boolean\",\n \"default\": true\n },\n \"tooltipColor\": {\n \"title\": \"Tooltip background color\",\n \"type\": \"string\",\n \"default\": \"#fff\"\n },\n \"tooltipFontColor\": {\n \"title\": \"Tooltip font color\",\n \"type\": \"string\",\n \"default\": \"#000\"\n },\n \"tooltipOpacity\": {\n \"title\": \"Tooltip opacity (0-1)\",\n \"type\": \"number\",\n \"default\": 1\n },\n \"tooltipPattern\": {\n \"title\": \"Tooltip (for ex. 'Text ${keyName} units.' or Link text')\",\n \"type\": \"string\",\n \"default\": \"${entityName}Latitude: ${latitude:7}Longitude: ${longitude:7}\"\n },\n \"useTooltipFunction\": {\n \"title\": \"Use tooltip function\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"tooltipFunction\": {\n \"title\": \"Tooltip function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"color\": {\n \"title\": \"Path color\",\n \"type\": \"string\"\n },\n \"strokeWeight\": {\n \"title\": \"Stroke weight\",\n \"type\": \"number\",\n \"default\": 2\n },\n \"strokeOpacity\": {\n \"title\": \"Stroke opacity\",\n \"type\": \"number\",\n \"default\": 1\n },\n \"useColorFunction\": {\n \"title\": \"Use path color function\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"colorFunction\": {\n \"title\": \"Path color function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"usePolylineDecorator\": {\n \"title\": \"Use path decorator\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"decoratorSymbol\": {\n \"title\": \"Decorator symbol\",\n \"type\": \"string\",\n \"default\": \"arrowHead\"\n },\n \"decoratorSymbolSize\": {\n \"title\": \"Decorator symbol size (px)\",\n \"type\": \"number\",\n \"default\": 10\n },\n \"useDecoratorCustomColor\": {\n \"title\": \"Use path decorator custom color\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"decoratorCustomColor\": {\n \"title\": \"Decorator custom color\",\n \"type\": \"string\",\n \"default\": \"#000\"\n },\n \"decoratorOffset\": {\n \"title\": \"Decorator offset\",\n \"type\": \"string\",\n \"default\": \"20px\"\n },\n \"endDecoratorOffset\": {\n \"title\": \"End decorator offset\",\n \"type\": \"string\",\n \"default\": \"20px\"\n },\n \"decoratorRepeat\": {\n \"title\": \"Decorator repeat\",\n \"type\": \"string\",\n \"default\": \"20px\"\n },\n \"showPolygon\": {\n \"title\": \"Show polygon\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"polygonTooltipPattern\": {\n \"title\": \"Tooltip (for ex. 'Text ${keyName} units.' or Link text')\",\n \"type\": \"string\",\n \"default\": \"${entityName}TimeStamp: ${ts:7}\"\n },\n \"usePolygonTooltipFunction\": {\n \"title\": \"Use polygon tooltip function\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"polygonTooltipFunction\": {\n \"title\": \"Polygon tooltip function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"polygonColor\": {\n \"title\": \"Polygon color\",\n \"type\": \"string\"\n },\n \"polygonOpacity\": {\n \"title\": \"Polygon opacity\",\n \"type\": \"number\",\n \"default\": 0.5\n },\n \"polygonStrokeColor\": {\n \"title\": \"Polygon border color\",\n \"type\": \"string\"\n },\n \"polygonStrokeOpacity\": {\n \"title\": \"Polygon border opacity\",\n \"type\": \"number\",\n \"default\": 1\n },\n \"polygonStrokeWeight\": {\n \"title\": \"Polygon border weight\",\n \"type\": \"number\",\n \"default\": 1\n },\n \"usePolygonColorFunction\": {\n \"title\": \"Use polygon color function\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"polygonColorFunction\": {\n \"title\": \"Polygon Color function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"showPoints\": {\n \"title\": \"Show points\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"pointColor\": {\n \"title\": \"Point color\",\n \"type\": \"string\"\n },\n \"pointSize\": {\n \"title\": \"Point size (px)\",\n \"type\": \"number\",\n \"default\": 10\n },\n \"usePointAsAnchor\": {\n \"title\": \"Use point as anchor\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"pointAsAnchorFunction\": {\n \"title\": \"Point as anchor function: f(data, dsData, dsIndex)\",\n \"type\": \"string\"\n },\n \"pointTooltipOnRightPanel\": {\n \"title\": \"Independant point tooltip\",\n \"type\": \"boolean\",\n \"default\": true\n },\n \"autocloseTooltip\": {\n \"title\": \"Auto-close point popup\",\n \"type\": \"boolean\",\n \"default\": true\n },\n \"markerImage\": {\n \"title\": \"Custom marker image\",\n \"type\": \"string\"\n },\n \"markerImageSize\": {\n \"title\": \"Custom marker image size (px)\",\n \"type\": \"number\",\n \"default\": 34\n },\n \"rotationAngle\": {\n \"title\": \"Set additional rotation angle for marker (deg)\",\n \"type\": \"number\",\n \"default\": 180\n },\n \"useMarkerImageFunction\":{\n \"title\":\"Use marker image function\",\n \"type\":\"boolean\",\n \"default\":false\n },\n \"markerImageFunction\":{\n \"title\":\"Marker image function: f(data, images, dsData, dsIndex)\",\n \"type\":\"string\"\n },\n \"markerImages\":{\n \"title\":\"Marker images\",\n \"type\":\"array\",\n \"items\":{\n \"title\":\"Marker image\",\n \"type\":\"string\"\n }\n }\n },\n \"required\": []\n },\n \"form\": [{\n \"key\": \"mapProvider\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [{\n \"value\": \"OpenStreetMap.Mapnik\",\n \"label\": \"OpenStreetMap.Mapnik (Default)\"\n }, {\n \"value\": \"OpenStreetMap.BlackAndWhite\",\n \"label\": \"OpenStreetMap.BlackAndWhite\"\n }, {\n \"value\": \"OpenStreetMap.HOT\",\n \"label\": \"OpenStreetMap.HOT\"\n }, {\n \"value\": \"Esri.WorldStreetMap\",\n \"label\": \"Esri.WorldStreetMap\"\n }, {\n \"value\": \"Esri.WorldTopoMap\",\n \"label\": \"Esri.WorldTopoMap\"\n }, {\n \"value\": \"CartoDB.Positron\",\n \"label\": \"CartoDB.Positron\"\n }, {\n \"value\": \"CartoDB.DarkMatter\",\n \"label\": \"CartoDB.DarkMatter\"\n }]\n }, \"normalizationStep\", \"latKeyName\", \"lngKeyName\", \"polKeyName\", \"showLabel\", \"label\", \"useLabelFunction\", {\n \"key\": \"labelFunction\",\n \"type\": \"javascript\"\n }, \"showTooltip\", {\n \"key\": \"tooltipColor\",\n \"type\": \"color\"\n }, {\n \"key\": \"tooltipFontColor\",\n \"type\": \"color\"\n },\"tooltipOpacity\", {\n \"key\": \"tooltipPattern\",\n \"type\": \"textarea\"\n }, \"useTooltipFunction\", {\n \"key\": \"tooltipFunction\",\n \"type\": \"javascript\"\n }, {\n \"key\": \"color\",\n \"type\": \"color\"\n }, \"useColorFunction\", {\n \"key\": \"colorFunction\",\n \"type\": \"javascript\"\n }, \"usePolylineDecorator\", {\n \"key\": \"decoratorSymbol\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [{\n \"value\": \"arrowHead\",\n \"label\": \"Arrow\"\n }, {\n \"value\": \"dash\",\n \"label\": \"Dash\"\n }]\n }, \"decoratorSymbolSize\", \"useDecoratorCustomColor\", {\n \"key\": \"decoratorCustomColor\",\n \"type\": \"color\"\n }, {\n \"key\": \"decoratorOffset\",\n \"type\": \"textarea\"\n },{\n \"key\": \"endDecoratorOffset\",\n \"type\": \"textarea\"\n }, {\n \"key\": \"decoratorRepeat\",\n \"type\": \"textarea\"\n }, \"strokeWeight\", \"strokeOpacity\", \"showPolygon\", {\n \"key\": \"polygonTooltipPattern\",\n \"type\": \"textarea\"\n },\"usePolygonTooltipFunction\", {\n \"key\": \"polygonTooltipFunction\",\n \"type\": \"javascript\"\n },{\n \"key\": \"polygonColor\",\n \"type\": \"color\"\n },\t\"polygonOpacity\", {\n \"key\": \"polygonStrokeColor\",\n \"type\": \"color\"\n },\t\"polygonStrokeOpacity\",\"polygonStrokeWeight\",\"usePolygonColorFunction\",\t{\n \"key\": \"polygonColorFunction\",\n \"type\": \"javascript\"\n },\"showPoints\",{\n \"key\": \"pointColor\",\n \"type\": \"color\"\n }, \"pointSize\",\"usePointAsAnchor\", {\n \"key\": \"pointAsAnchorFunction\",\n \"type\": \"javascript\"\n },\"pointTooltipOnRightPanel\", \"autocloseTooltip\", {\n \"key\": \"markerImage\",\n \"type\": \"image\"\n }, \"markerImageSize\", \"rotationAngle\",\"useMarkerImageFunction\",\n {\n \"key\":\"markerImageFunction\",\n \"type\":\"javascript\"\n }, {\n \"key\":\"markerImages\",\n \"items\":[\n {\n \"key\":\"markerImages[]\",\n \"type\":\"image\"\n }\n ]\n }]\n}", "dataKeySettingsSchema": "{}", "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"latitude\",\"color\":\"#2196f3\",\"settings\":{\"showLines\":true,\"fillLines\":true,\"showPoints\":false},\"_hash\":0.8587686344902596,\"funcBody\":\"var gpsData = [\\n\\t37.771210000,-122.510960000,\\n\\t37.771340000,-122.510980000,\\n\\t37.771340000,-122.510980000,\\n\\t37.771360000,-122.510850000,\\n\\t37.771380000,-122.510550000,\\n\\t37.771400000,-122.509900000,\\n\\t37.771410000,-122.509660000,\\n\\t37.771430000,-122.509360000,\\n\\t37.771430000,-122.509270000,\\n\\t37.771450000,-122.508840000,\\n\\t37.771490000,-122.507880000,\\n\\t37.771490000,-122.507780000,\\n\\t37.771530000,-122.507140000,\\n\\t37.771550000,-122.506690000,\\n\\t37.771560000,-122.506310000,\\n\\t37.771600000,-122.505640000,\\n\\t37.771650000,-122.504540000,\\n\\t37.771670000,-122.503990000,\\n\\t37.771700000,-122.503490000,\\n\\t37.771740000,-122.502430000,\\n\\t37.771790000,-122.501360000,\\n\\t37.771840000,-122.500290000,\\n\\t37.771870000,-122.499730000,\\n\\t37.771890000,-122.499210000,\\n\\t37.771940000,-122.498140000,\\n\\t37.771990000,-122.497070000,\\n\\t37.772000000,-122.496690000,\\n\\t37.772020000,-122.496350000,\\n\\t37.772030000,-122.496110000,\\n\\t37.772040000,-122.496000000,\\n\\t37.772040000,-122.495890000,\\n\\t37.772060000,-122.495440000,\\n\\t37.772090000,-122.494930000,\\n\\t37.772120000,-122.494160000,\\n\\t37.772130000,-122.493860000,\\n\\t37.772180000,-122.492790000,\\n\\t37.772200000,-122.492300000,\\n\\t37.772220000,-122.491840000,\\n\\t37.772230000,-122.491710000,\\n\\t37.772280000,-122.490630000,\\n\\t37.772330000,-122.489560000,\\n\\t37.772330000,-122.489470000,\\n\\t37.772360000,-122.489030000,\\n\\t37.772380000,-122.488490000,\\n\\t37.772430000,-122.487420000,\\n\\t37.772450000,-122.486980000,\\n\\t37.772480000,-122.486360000,\\n\\t37.772520000,-122.485280000,\\n\\t37.772560000,-122.484400000,\\n\\t37.772570000,-122.484300000,\\n\\t37.772570000,-122.484150000,\\n\\t37.772620000,-122.483140000,\\n\\t37.772680000,-122.482050000,\\n\\t37.772700000,-122.481370000,\\n\\t37.772710000,-122.481000000,\\n\\t37.772730000,-122.480740000,\\n\\t37.772770000,-122.479930000,\\n\\t37.772820000,-122.478860000,\\n\\t37.772870000,-122.477790000,\\n\\t37.772900000,-122.477110000,\\n\\t37.772920000,-122.476710000,\\n\\t37.772960000,-122.475650000,\\n\\t37.772990000,-122.474950000,\\n\\t37.773010000,-122.474580000,\\n\\t37.773060000,-122.473450000,\\n\\t37.773120000,-122.472330000,\\n\\t37.773140000,-122.471850000,\\n\\t37.773140000,-122.471730000,\\n\\t37.773150000,-122.471640000,\\n\\t37.773170000,-122.471260000,\\n\\t37.773190000,-122.470570000,\\n\\t37.773210000,-122.470190000,\\n\\t37.773230000,-122.469770000,\\n\\t37.773250000,-122.469370000,\\n\\t37.773260000,-122.469120000,\\n\\t37.773290000,-122.468490000,\\n\\t37.773300000,-122.468150000,\\n\\t37.773310000,-122.468050000,\\n\\t37.773310000,-122.467940000,\\n\\t37.773320000,-122.467740000,\\n\\t37.773350000,-122.467270000,\\n\\t37.773360000,-122.466980000,\\n\\t37.773360000,-122.466870000,\\n\\t37.773370000,-122.466610000,\\n\\t37.773390000,-122.466300000,\\n\\t37.773400000,-122.466000000,\\n\\t37.773400000,-122.465910000,\\n\\t37.773410000,-122.465790000,\\n\\t37.773430000,-122.465520000,\\n\\t37.773460000,-122.465210000,\\n\\t37.773490000,-122.464980000,\\n\\t37.773500000,-122.464910000,\\n\\t37.773460000,-122.464830000,\\n\\t37.773560000,-122.464070000,\\n\\t37.773580000,-122.463900000,\\n\\t37.773590000,-122.463810000,\\n\\t37.773600000,-122.463780000,\\n\\t37.773610000,-122.463670000,\\n\\t37.773660000,-122.463320000,\\n\\t37.773740000,-122.462700000,\\n\\t37.773770000,-122.462440000,\\n\\t37.773860000,-122.461730000,\\n\\t37.773870000,-122.461640000,\\n\\t37.773920000,-122.461260000,\\n\\t37.773970000,-122.460890000,\\n\\t37.774010000,-122.460570000,\\n\\t37.774110000,-122.459760000,\\n\\t37.774140000,-122.459490000,\\n\\t37.774270000,-122.458520000,\\n\\t37.774270000,-122.458440000,\\n\\t37.774270000,-122.458380000,\\n\\t37.774320000,-122.458270000,\\n\\t37.774340000,-122.458050000,\\n\\t37.774510000,-122.456680000,\\n\\t37.774560000,-122.456310000,\\n\\t37.774700000,-122.455280000,\\n\\t37.774760000,-122.454780000,\\n\\t37.774770000,-122.454670000,\\n\\t37.774770000,-122.454670000,\\n\\t37.774670000,-122.454650000,\\n\\t37.774670000,-122.454650000,\\n\\t37.774580000,-122.454640000,\\n\\t37.774300000,-122.454580000,\\n\\t37.774190000,-122.454560000,\\n\\t37.773700000,-122.454460000,\\n\\t37.772910000,-122.454310000,\\n\\t37.772620000,-122.454260000,\\n\\t37.772430000,-122.454220000,\\n\\t37.771980000,-122.454110000,\\n\\t37.771910000,-122.454100000,\\n\\t37.771760000,-122.454060000,\\n\\t37.771690000,-122.454050000,\\n\\t37.771620000,-122.454030000,\\n\\t37.771530000,-122.454010000,\\n\\t37.771380000,-122.453970000,\\n\\t37.771250000,-122.453950000,\\n\\t37.771100000,-122.453930000,\\n\\t37.771020000,-122.453920000,\\n\\t37.770920000,-122.453900000,\\n\\t37.770810000,-122.453890000,\\n\\t37.770660000,-122.453860000,\\n\\t37.770110000,-122.453750000,\\n\\t37.769560000,-122.453640000,\\n\\t37.769360000,-122.453600000,\\n\\t37.769250000,-122.453580000,\\n\\t37.769180000,-122.453560000,\\n\\t37.769090000,-122.453540000,\\n\\t37.768780000,-122.453480000,\\n\\t37.768250000,-122.453380000,\\n\\t37.768160000,-122.453360000,\\n\\t37.767820000,-122.453290000,\\n\\t37.767310000,-122.453190000,\\n\\t37.767160000,-122.453160000,\\n\\t37.767010000,-122.453130000,\\n\\t37.766760000,-122.453070000,\\n\\t37.766550000,-122.453030000,\\n\\t37.766550000,-122.453030000,\\n\\t37.766390000,-122.452990000,\\n\\t37.766390000,-122.452990000,\\n\\t37.766290000,-122.453720000,\\n\\t37.766180000,-122.454610000,\\n\\t37.766130000,-122.454980000,\\n\\t37.765960000,-122.456290000,\\n\\t37.765960000,-122.456340000,\\n\\t37.765960000,-122.456360000,\\n\\t37.765960000,-122.456380000,\\n\\t37.765960000,-122.456410000,\\n\\t37.765960000,-122.456460000,\\n\\t37.765940000,-122.456630000,\\n\\t37.765930000,-122.456700000,\\n\\t37.765920000,-122.456810000,\\n\\t37.765910000,-122.456930000,\\n\\t37.765910000,-122.457020000,\\n\\t37.765920000,-122.457160000,\\n\\t37.765930000,-122.457270000,\\n\\t37.765940000,-122.457360000,\\n\\t37.765950000,-122.457410000,\\n\\t37.765960000,-122.457470000,\\n\\t37.765980000,-122.457560000,\\n\\t37.766010000,-122.457660000,\\n\\t37.766070000,-122.457830000,\\n\\t37.766070000,-122.457830000,\\n\\t37.766120000,-122.457980000,\\n\\t37.766180000,-122.458180000,\\n\\t37.766190000,-122.458200000,\\n\\t37.766240000,-122.458400000,\\n\\t37.766270000,-122.458530000,\\n\\t37.766290000,-122.458600000,\\n\\t37.766300000,-122.458690000,\\n\\t37.766300000,-122.458880000,\\n\\t37.766300000,-122.458970000,\\n\\t37.766280000,-122.459470000,\\n\\t37.766270000,-122.459520000,\\n\\t37.766270000,-122.459560000,\\n\\t37.766280000,-122.459600000,\\n\\t37.766280000,-122.459630000,\\n\\t37.766290000,-122.459670000,\\n\\t37.766300000,-122.459700000,\\n\\t37.766310000,-122.459730000,\\n\\t37.766320000,-122.459750000,\\n\\t37.766330000,-122.459770000,\\n\\t37.766350000,-122.459800000,\\n\\t37.766390000,-122.459860000,\\n\\t37.766340000,-122.459970000,\\n\\t37.766290000,-122.460150000,\\n\\t37.766290000,-122.460230000,\\n\\t37.766280000,-122.460280000,\\n\\t37.766260000,-122.460330000,\\n\\t37.766250000,-122.460420000,\\n\\t37.766240000,-122.460520000,\\n\\t37.766230000,-122.460670000,\\n\\t37.766230000,-122.460800000,\\n\\t37.766230000,-122.460900000,\\n\\t37.766210000,-122.461110000,\\n\\t37.766170000,-122.462030000,\\n\\t37.766160000,-122.462170000,\\n\\t37.766150000,-122.462520000,\\n\\t37.766120000,-122.463260000,\\n\\t37.766090000,-122.464130000,\\n\\t37.766070000,-122.464350000,\\n\\t37.766070000,-122.464440000,\\n\\t37.766060000,-122.464620000,\\n\\t37.766030000,-122.465400000,\\n\\t37.765980000,-122.466470000,\\n\\t37.765940000,-122.467530000,\\n\\t37.765930000,-122.467680000,\\n\\t37.765890000,-122.468600000,\\n\\t37.765860000,-122.468980000,\\n\\t37.765830000,-122.469660000,\\n\\t37.765780000,-122.470740000,\\n\\t37.765770000,-122.471030000,\\n\\t37.765770000,-122.471140000,\\n\\t37.765760000,-122.471380000,\\n\\t37.765740000,-122.471820000,\\n\\t37.765690000,-122.472950000,\\n\\t37.765680000,-122.473220000,\\n\\t37.765670000,-122.473320000,\\n\\t37.765640000,-122.474070000,\\n\\t37.765590000,-122.475140000,\\n\\t37.765590000,-122.475400000,\\n\\t37.765580000,-122.475520000,\\n\\t37.765550000,-122.476200000,\\n\\t37.765500000,-122.477180000,\\n\\t37.765500000,-122.477280000,\\n\\t37.765490000,-122.477400000,\\n\\t37.765490000,-122.477440000,\\n\\t37.765490000,-122.477440000,\\n\\t37.765490000,-122.477490000,\\n\\t37.765470000,-122.477770000,\\n\\t37.765450000,-122.478340000,\\n\\t37.765450000,-122.478420000,\\n\\t37.765430000,-122.478870000,\\n\\t37.765400000,-122.479430000,\\n\\t37.765380000,-122.479810000,\\n\\t37.765350000,-122.480480000,\\n\\t37.765300000,-122.481570000,\\n\\t37.765300000,-122.481660000,\\n\\t37.765290000,-122.481950000,\\n\\t37.765260000,-122.482640000,\\n\\t37.765220000,-122.483570000,\\n\\t37.765210000,-122.483710000,\\n\\t37.765210000,-122.483810000,\\n\\t37.765190000,-122.484130000,\\n\\t37.765160000,-122.484780000,\\n\\t37.765120000,-122.485860000,\\n\\t37.765110000,-122.485960000,\\n\\t37.765100000,-122.486170000,\\n\\t37.765070000,-122.486930000,\\n\\t37.765020000,-122.488000000,\\n\\t37.765020000,-122.488100000,\\n\\t37.765000000,-122.488360000,\\n\\t37.764980000,-122.488970000,\\n\\t37.764970000,-122.489070000,\\n\\t37.764930000,-122.490150000,\\n\\t37.764920000,-122.490240000,\\n\\t37.764910000,-122.490490000,\\n\\t37.764880000,-122.491210000,\\n\\t37.764830000,-122.492290000,\\n\\t37.764790000,-122.493260000,\\n\\t37.764780000,-122.493350000,\\n\\t37.764740000,-122.494420000,\\n\\t37.764720000,-122.494730000,\\n\\t37.764710000,-122.495500000,\\n\\t37.764700000,-122.495630000,\\n\\t37.764690000,-122.495940000,\\n\\t37.764680000,-122.496260000,\\n\\t37.764670000,-122.496480000,\\n\\t37.764600000,-122.497490000,\\n\\t37.764590000,-122.497650000,\\n\\t37.764550000,-122.498720000,\\n\\t37.764500000,-122.499780000,\\n\\t37.764450000,-122.500870000,\\n\\t37.764440000,-122.501060000,\\n\\t37.764400000,-122.501930000,\\n\\t37.764360000,-122.502990000,\\n\\t37.764310000,-122.504080000,\\n\\t37.764260000,-122.505140000,\\n\\t37.764260000,-122.505250000,\\n\\t37.764220000,-122.506220000,\\n\\t37.764170000,-122.507280000,\\n\\t37.764120000,-122.508360000,\\n\\t37.764100000,-122.508850000,\\n\\t37.764100000,-122.509150000,\\n\\t37.764100000,-122.509440000,\\n\\t37.764090000,-122.509760000,\\n\\t37.764080000,-122.510200000,\\n\\t37.764070000,-122.510320000,\\n\\t37.764060000,-122.510430000,\\n\\t37.764030000,-122.510430000,\\n\\t37.764030000,-122.510430000,\\n\\t37.763960000,-122.510430000,\\n\\t37.763960000,-122.510290000,\\n\\t37.764070000,-122.510320000,\\n\\t37.764170000,-122.510320000,\\n\\t37.764410000,-122.510340000,\\n\\t37.764610000,-122.510350000,\\n\\t37.764780000,-122.510330000,\\n\\t37.764960000,-122.510310000,\\n\\t37.765340000,-122.510280000,\\n\\t37.765570000,-122.510260000,\\n\\t37.765840000,-122.510250000,\\n\\t37.765900000,-122.510250000,\\n\\t37.766140000,-122.510260000,\\n\\t37.766410000,-122.510260000,\\n\\t37.766620000,-122.510270000,\\n\\t37.767040000,-122.510310000,\\n\\t37.767270000,-122.510330000,\\n\\t37.767620000,-122.510390000,\\n\\t37.767670000,-122.510390000,\\n\\t37.767740000,-122.510410000,\\n\\t37.767840000,-122.510430000,\\n\\t37.768280000,-122.510530000,\\n\\t37.768620000,-122.510600000,\\n\\t37.768690000,-122.510620000,\\n\\t37.769020000,-122.510690000,\\n\\t37.769260000,-122.510730000,\\n\\t37.769420000,-122.510750000,\\n\\t37.770010000,-122.510830000,\\n\\t37.770340000,-122.510860000,\\n\\t37.770460000,-122.510870000,\\n\\t37.770930000,-122.510930000,\\n\\t37.770980000,-122.510930000,\\n];\\n let value = gpsData.indexOf(prevValue); \\nreturn gpsData[(value == -1 ? 0 : value + 2)];\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"longitude\",\"color\":\"#ffc107\",\"settings\":{\"showLines\":true,\"fillLines\":false,\"showPoints\":false},\"_hash\":0.12775350966079668,\"funcBody\":\"var gpsData = [\\n\\t37.771210000,-122.510960000,\\n\\t37.771340000,-122.510980000,\\n\\t37.771340000,-122.510980000,\\n\\t37.771360000,-122.510850000,\\n\\t37.771380000,-122.510550000,\\n\\t37.771400000,-122.509900000,\\n\\t37.771410000,-122.509660000,\\n\\t37.771430000,-122.509360000,\\n\\t37.771430000,-122.509270000,\\n\\t37.771450000,-122.508840000,\\n\\t37.771490000,-122.507880000,\\n\\t37.771490000,-122.507780000,\\n\\t37.771530000,-122.507140000,\\n\\t37.771550000,-122.506690000,\\n\\t37.771560000,-122.506310000,\\n\\t37.771600000,-122.505640000,\\n\\t37.771650000,-122.504540000,\\n\\t37.771670000,-122.503990000,\\n\\t37.771700000,-122.503490000,\\n\\t37.771740000,-122.502430000,\\n\\t37.771790000,-122.501360000,\\n\\t37.771840000,-122.500290000,\\n\\t37.771870000,-122.499730000,\\n\\t37.771890000,-122.499210000,\\n\\t37.771940000,-122.498140000,\\n\\t37.771990000,-122.497070000,\\n\\t37.772000000,-122.496690000,\\n\\t37.772020000,-122.496350000,\\n\\t37.772030000,-122.496110000,\\n\\t37.772040000,-122.496000000,\\n\\t37.772040000,-122.495890000,\\n\\t37.772060000,-122.495440000,\\n\\t37.772090000,-122.494930000,\\n\\t37.772120000,-122.494160000,\\n\\t37.772130000,-122.493860000,\\n\\t37.772180000,-122.492790000,\\n\\t37.772200000,-122.492300000,\\n\\t37.772220000,-122.491840000,\\n\\t37.772230000,-122.491710000,\\n\\t37.772280000,-122.490630000,\\n\\t37.772330000,-122.489560000,\\n\\t37.772330000,-122.489470000,\\n\\t37.772360000,-122.489030000,\\n\\t37.772380000,-122.488490000,\\n\\t37.772430000,-122.487420000,\\n\\t37.772450000,-122.486980000,\\n\\t37.772480000,-122.486360000,\\n\\t37.772520000,-122.485280000,\\n\\t37.772560000,-122.484400000,\\n\\t37.772570000,-122.484300000,\\n\\t37.772570000,-122.484150000,\\n\\t37.772620000,-122.483140000,\\n\\t37.772680000,-122.482050000,\\n\\t37.772700000,-122.481370000,\\n\\t37.772710000,-122.481000000,\\n\\t37.772730000,-122.480740000,\\n\\t37.772770000,-122.479930000,\\n\\t37.772820000,-122.478860000,\\n\\t37.772870000,-122.477790000,\\n\\t37.772900000,-122.477110000,\\n\\t37.772920000,-122.476710000,\\n\\t37.772960000,-122.475650000,\\n\\t37.772990000,-122.474950000,\\n\\t37.773010000,-122.474580000,\\n\\t37.773060000,-122.473450000,\\n\\t37.773120000,-122.472330000,\\n\\t37.773140000,-122.471850000,\\n\\t37.773140000,-122.471730000,\\n\\t37.773150000,-122.471640000,\\n\\t37.773170000,-122.471260000,\\n\\t37.773190000,-122.470570000,\\n\\t37.773210000,-122.470190000,\\n\\t37.773230000,-122.469770000,\\n\\t37.773250000,-122.469370000,\\n\\t37.773260000,-122.469120000,\\n\\t37.773290000,-122.468490000,\\n\\t37.773300000,-122.468150000,\\n\\t37.773310000,-122.468050000,\\n\\t37.773310000,-122.467940000,\\n\\t37.773320000,-122.467740000,\\n\\t37.773350000,-122.467270000,\\n\\t37.773360000,-122.466980000,\\n\\t37.773360000,-122.466870000,\\n\\t37.773370000,-122.466610000,\\n\\t37.773390000,-122.466300000,\\n\\t37.773400000,-122.466000000,\\n\\t37.773400000,-122.465910000,\\n\\t37.773410000,-122.465790000,\\n\\t37.773430000,-122.465520000,\\n\\t37.773460000,-122.465210000,\\n\\t37.773490000,-122.464980000,\\n\\t37.773500000,-122.464910000,\\n\\t37.773460000,-122.464830000,\\n\\t37.773560000,-122.464070000,\\n\\t37.773580000,-122.463900000,\\n\\t37.773590000,-122.463810000,\\n\\t37.773600000,-122.463780000,\\n\\t37.773610000,-122.463670000,\\n\\t37.773660000,-122.463320000,\\n\\t37.773740000,-122.462700000,\\n\\t37.773770000,-122.462440000,\\n\\t37.773860000,-122.461730000,\\n\\t37.773870000,-122.461640000,\\n\\t37.773920000,-122.461260000,\\n\\t37.773970000,-122.460890000,\\n\\t37.774010000,-122.460570000,\\n\\t37.774110000,-122.459760000,\\n\\t37.774140000,-122.459490000,\\n\\t37.774270000,-122.458520000,\\n\\t37.774270000,-122.458440000,\\n\\t37.774270000,-122.458380000,\\n\\t37.774320000,-122.458270000,\\n\\t37.774340000,-122.458050000,\\n\\t37.774510000,-122.456680000,\\n\\t37.774560000,-122.456310000,\\n\\t37.774700000,-122.455280000,\\n\\t37.774760000,-122.454780000,\\n\\t37.774770000,-122.454670000,\\n\\t37.774770000,-122.454670000,\\n\\t37.774670000,-122.454650000,\\n\\t37.774670000,-122.454650000,\\n\\t37.774580000,-122.454640000,\\n\\t37.774300000,-122.454580000,\\n\\t37.774190000,-122.454560000,\\n\\t37.773700000,-122.454460000,\\n\\t37.772910000,-122.454310000,\\n\\t37.772620000,-122.454260000,\\n\\t37.772430000,-122.454220000,\\n\\t37.771980000,-122.454110000,\\n\\t37.771910000,-122.454100000,\\n\\t37.771760000,-122.454060000,\\n\\t37.771690000,-122.454050000,\\n\\t37.771620000,-122.454030000,\\n\\t37.771530000,-122.454010000,\\n\\t37.771380000,-122.453970000,\\n\\t37.771250000,-122.453950000,\\n\\t37.771100000,-122.453930000,\\n\\t37.771020000,-122.453920000,\\n\\t37.770920000,-122.453900000,\\n\\t37.770810000,-122.453890000,\\n\\t37.770660000,-122.453860000,\\n\\t37.770110000,-122.453750000,\\n\\t37.769560000,-122.453640000,\\n\\t37.769360000,-122.453600000,\\n\\t37.769250000,-122.453580000,\\n\\t37.769180000,-122.453560000,\\n\\t37.769090000,-122.453540000,\\n\\t37.768780000,-122.453480000,\\n\\t37.768250000,-122.453380000,\\n\\t37.768160000,-122.453360000,\\n\\t37.767820000,-122.453290000,\\n\\t37.767310000,-122.453190000,\\n\\t37.767160000,-122.453160000,\\n\\t37.767010000,-122.453130000,\\n\\t37.766760000,-122.453070000,\\n\\t37.766550000,-122.453030000,\\n\\t37.766550000,-122.453030000,\\n\\t37.766390000,-122.452990000,\\n\\t37.766390000,-122.452990000,\\n\\t37.766290000,-122.453720000,\\n\\t37.766180000,-122.454610000,\\n\\t37.766130000,-122.454980000,\\n\\t37.765960000,-122.456290000,\\n\\t37.765960000,-122.456340000,\\n\\t37.765960000,-122.456360000,\\n\\t37.765960000,-122.456380000,\\n\\t37.765960000,-122.456410000,\\n\\t37.765960000,-122.456460000,\\n\\t37.765940000,-122.456630000,\\n\\t37.765930000,-122.456700000,\\n\\t37.765920000,-122.456810000,\\n\\t37.765910000,-122.456930000,\\n\\t37.765910000,-122.457020000,\\n\\t37.765920000,-122.457160000,\\n\\t37.765930000,-122.457270000,\\n\\t37.765940000,-122.457360000,\\n\\t37.765950000,-122.457410000,\\n\\t37.765960000,-122.457470000,\\n\\t37.765980000,-122.457560000,\\n\\t37.766010000,-122.457660000,\\n\\t37.766070000,-122.457830000,\\n\\t37.766070000,-122.457830000,\\n\\t37.766120000,-122.457980000,\\n\\t37.766180000,-122.458180000,\\n\\t37.766190000,-122.458200000,\\n\\t37.766240000,-122.458400000,\\n\\t37.766270000,-122.458530000,\\n\\t37.766290000,-122.458600000,\\n\\t37.766300000,-122.458690000,\\n\\t37.766300000,-122.458880000,\\n\\t37.766300000,-122.458970000,\\n\\t37.766280000,-122.459470000,\\n\\t37.766270000,-122.459520000,\\n\\t37.766270000,-122.459560000,\\n\\t37.766280000,-122.459600000,\\n\\t37.766280000,-122.459630000,\\n\\t37.766290000,-122.459670000,\\n\\t37.766300000,-122.459700000,\\n\\t37.766310000,-122.459730000,\\n\\t37.766320000,-122.459750000,\\n\\t37.766330000,-122.459770000,\\n\\t37.766350000,-122.459800000,\\n\\t37.766390000,-122.459860000,\\n\\t37.766340000,-122.459970000,\\n\\t37.766290000,-122.460150000,\\n\\t37.766290000,-122.460230000,\\n\\t37.766280000,-122.460280000,\\n\\t37.766260000,-122.460330000,\\n\\t37.766250000,-122.460420000,\\n\\t37.766240000,-122.460520000,\\n\\t37.766230000,-122.460670000,\\n\\t37.766230000,-122.460800000,\\n\\t37.766230000,-122.460900000,\\n\\t37.766210000,-122.461110000,\\n\\t37.766170000,-122.462030000,\\n\\t37.766160000,-122.462170000,\\n\\t37.766150000,-122.462520000,\\n\\t37.766120000,-122.463260000,\\n\\t37.766090000,-122.464130000,\\n\\t37.766070000,-122.464350000,\\n\\t37.766070000,-122.464440000,\\n\\t37.766060000,-122.464620000,\\n\\t37.766030000,-122.465400000,\\n\\t37.765980000,-122.466470000,\\n\\t37.765940000,-122.467530000,\\n\\t37.765930000,-122.467680000,\\n\\t37.765890000,-122.468600000,\\n\\t37.765860000,-122.468980000,\\n\\t37.765830000,-122.469660000,\\n\\t37.765780000,-122.470740000,\\n\\t37.765770000,-122.471030000,\\n\\t37.765770000,-122.471140000,\\n\\t37.765760000,-122.471380000,\\n\\t37.765740000,-122.471820000,\\n\\t37.765690000,-122.472950000,\\n\\t37.765680000,-122.473220000,\\n\\t37.765670000,-122.473320000,\\n\\t37.765640000,-122.474070000,\\n\\t37.765590000,-122.475140000,\\n\\t37.765590000,-122.475400000,\\n\\t37.765580000,-122.475520000,\\n\\t37.765550000,-122.476200000,\\n\\t37.765500000,-122.477180000,\\n\\t37.765500000,-122.477280000,\\n\\t37.765490000,-122.477400000,\\n\\t37.765490000,-122.477440000,\\n\\t37.765490000,-122.477440000,\\n\\t37.765490000,-122.477490000,\\n\\t37.765470000,-122.477770000,\\n\\t37.765450000,-122.478340000,\\n\\t37.765450000,-122.478420000,\\n\\t37.765430000,-122.478870000,\\n\\t37.765400000,-122.479430000,\\n\\t37.765380000,-122.479810000,\\n\\t37.765350000,-122.480480000,\\n\\t37.765300000,-122.481570000,\\n\\t37.765300000,-122.481660000,\\n\\t37.765290000,-122.481950000,\\n\\t37.765260000,-122.482640000,\\n\\t37.765220000,-122.483570000,\\n\\t37.765210000,-122.483710000,\\n\\t37.765210000,-122.483810000,\\n\\t37.765190000,-122.484130000,\\n\\t37.765160000,-122.484780000,\\n\\t37.765120000,-122.485860000,\\n\\t37.765110000,-122.485960000,\\n\\t37.765100000,-122.486170000,\\n\\t37.765070000,-122.486930000,\\n\\t37.765020000,-122.488000000,\\n\\t37.765020000,-122.488100000,\\n\\t37.765000000,-122.488360000,\\n\\t37.764980000,-122.488970000,\\n\\t37.764970000,-122.489070000,\\n\\t37.764930000,-122.490150000,\\n\\t37.764920000,-122.490240000,\\n\\t37.764910000,-122.490490000,\\n\\t37.764880000,-122.491210000,\\n\\t37.764830000,-122.492290000,\\n\\t37.764790000,-122.493260000,\\n\\t37.764780000,-122.493350000,\\n\\t37.764740000,-122.494420000,\\n\\t37.764720000,-122.494730000,\\n\\t37.764710000,-122.495500000,\\n\\t37.764700000,-122.495630000,\\n\\t37.764690000,-122.495940000,\\n\\t37.764680000,-122.496260000,\\n\\t37.764670000,-122.496480000,\\n\\t37.764600000,-122.497490000,\\n\\t37.764590000,-122.497650000,\\n\\t37.764550000,-122.498720000,\\n\\t37.764500000,-122.499780000,\\n\\t37.764450000,-122.500870000,\\n\\t37.764440000,-122.501060000,\\n\\t37.764400000,-122.501930000,\\n\\t37.764360000,-122.502990000,\\n\\t37.764310000,-122.504080000,\\n\\t37.764260000,-122.505140000,\\n\\t37.764260000,-122.505250000,\\n\\t37.764220000,-122.506220000,\\n\\t37.764170000,-122.507280000,\\n\\t37.764120000,-122.508360000,\\n\\t37.764100000,-122.508850000,\\n\\t37.764100000,-122.509150000,\\n\\t37.764100000,-122.509440000,\\n\\t37.764090000,-122.509760000,\\n\\t37.764080000,-122.510200000,\\n\\t37.764070000,-122.510320000,\\n\\t37.764060000,-122.510430000,\\n\\t37.764030000,-122.510430000,\\n\\t37.764030000,-122.510430000,\\n\\t37.763960000,-122.510430000,\\n\\t37.763960000,-122.510290000,\\n\\t37.764070000,-122.510320000,\\n\\t37.764170000,-122.510320000,\\n\\t37.764410000,-122.510340000,\\n\\t37.764610000,-122.510350000,\\n\\t37.764780000,-122.510330000,\\n\\t37.764960000,-122.510310000,\\n\\t37.765340000,-122.510280000,\\n\\t37.765570000,-122.510260000,\\n\\t37.765840000,-122.510250000,\\n\\t37.765900000,-122.510250000,\\n\\t37.766140000,-122.510260000,\\n\\t37.766410000,-122.510260000,\\n\\t37.766620000,-122.510270000,\\n\\t37.767040000,-122.510310000,\\n\\t37.767270000,-122.510330000,\\n\\t37.767620000,-122.510390000,\\n\\t37.767670000,-122.510390000,\\n\\t37.767740000,-122.510410000,\\n\\t37.767840000,-122.510430000,\\n\\t37.768280000,-122.510530000,\\n\\t37.768620000,-122.510600000,\\n\\t37.768690000,-122.510620000,\\n\\t37.769020000,-122.510690000,\\n\\t37.769260000,-122.510730000,\\n\\t37.769420000,-122.510750000,\\n\\t37.770010000,-122.510830000,\\n\\t37.770340000,-122.510860000,\\n\\t37.770460000,-122.510870000,\\n\\t37.770930000,-122.510930000,\\n\\t37.770980000,-122.510930000,\\n];\\n let value = gpsData.indexOf(prevValue); \\nreturn gpsData[(value == -1 ? 1 : value + 2)];\"}]}],\"timewindow\":{\"history\":{\"interval\":1000,\"timewindowMs\":60000},\"aggregation\":{\"type\":\"NONE\",\"limit\":500}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{\"mapProvider\":\"OpenStreetMap.Mapnik\",\"latKeyName\":\"latitude\",\"lngKeyName\":\"longitude\",\"showLabel\":true,\"label\":\"${entityName}\",\"showTooltip\":true,\"tooltipColor\":\"#fff\",\"tooltipFontColor\":\"#000\",\"tooltipOpacity\":1,\"tooltipPattern\":\"${entityName}Latitude: ${latitude:7}Longitude: ${longitude:7}End Time: ${maxTime}Start Time: ${minTime}\",\"strokeWeight\":2,\"strokeOpacity\":1,\"pointSize\":10,\"markerImageSize\":34,\"rotationAngle\":180},\"title\":\"Trip Animation\",\"dropShadow\":true,\"enableFullscreen\":true,\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"mobileHeight\":null,\"widgetStyle\":{},\"useDashboardTimewindow\":false,\"showLegend\":false,\"actions\":{},\"legendConfig\":{\"position\":\"bottom\",\"showMin\":false,\"showMax\":false,\"showAvg\":false,\"showTotal\":false}}" } } ] -} \ No newline at end of file +} diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql new file mode 100644 index 0000000000..0916c241a1 --- /dev/null +++ b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql @@ -0,0 +1,123 @@ +-- +-- Copyright © 2016-2020 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + max_tenant_ttl bigint; + max_customer_ttl bigint; + max_ttl bigint; + date timestamp; + partition_by_max_ttl_date varchar; + partition_month varchar; + partition_day varchar; + partition_year varchar; + partition varchar; + partition_to_delete varchar; + + +BEGIN + SELECT max(attribute_kv.long_v) + FROM tenant + INNER JOIN attribute_kv ON tenant.id = attribute_kv.entity_id + WHERE attribute_kv.attribute_key = 'TTL' + into max_tenant_ttl; + SELECT max(attribute_kv.long_v) + FROM customer + INNER JOIN attribute_kv ON customer.id = attribute_kv.entity_id + WHERE attribute_kv.attribute_key = 'TTL' + into max_customer_ttl; + max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl); + if max_ttl IS NOT NULL AND max_ttl > 0 THEN + date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000)); + partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date); + RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date; + IF partition_by_max_ttl_date IS NOT NULL THEN + CASE + WHEN partition_type = 'DAYS' THEN + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4); + partition_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5); + WHEN partition_type = 'MONTHS' THEN + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4); + ELSE + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + END CASE; + FOR partition IN SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + AND tablename like 'ts_kv_' || '%' + AND tablename != 'ts_kv_latest' + AND tablename != 'ts_kv_dictionary' + LOOP + IF partition != partition_by_max_ttl_date THEN + IF partition_year IS NOT NULL THEN + IF SPLIT_PART(partition, '_', 3)::integer < partition_year::integer THEN + partition_to_delete := partition; + ELSE + IF partition_month IS NOT NULL THEN + IF SPLIT_PART(partition, '_', 4)::integer < partition_month::integer THEN + partition_to_delete := partition; + ELSE + IF partition_day IS NOT NULL THEN + IF SPLIT_PART(partition, '_', 5)::integer < partition_day::integer THEN + partition_to_delete := partition; + END IF; + END IF; + END IF; + END IF; + END IF; + END IF; + END IF; + IF partition_to_delete IS NOT NULL THEN + RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete; + EXECUTE format('DROP TABLE %I', partition_to_delete); + deleted := deleted + 1; + END IF; + END LOOP; + END IF; + END IF; +END +$$; + +CREATE OR REPLACE FUNCTION get_partition_by_max_ttl_date(IN partition_type varchar, IN date timestamp, OUT partition varchar) AS +$$ +BEGIN + CASE + WHEN partition_type = 'DAYS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM') || '_' || to_char(date, 'dd'); + WHEN partition_type = 'MONTHS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM'); + WHEN partition_type = 'YEARS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy'); + WHEN partition_type = 'INDEFINITE' THEN + partition := NULL; + ELSE + partition := NULL; + END CASE; + IF partition IS NOT NULL THEN + IF NOT EXISTS(SELECT + FROM pg_tables + WHERE schemaname = 'public' + AND tablename = partition) THEN + partition := NULL; + RAISE NOTICE 'Failed to found partition by ttl'; + END IF; + END IF; +END; +$$ LANGUAGE plpgsql; diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql b/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql new file mode 100644 index 0000000000..671d39aae5 --- /dev/null +++ b/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql @@ -0,0 +1,261 @@ +-- +-- Copyright © 2016-2020 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- call create_partition_ts_kv_table(); + +CREATE OR REPLACE PROCEDURE create_partition_ts_kv_table() LANGUAGE plpgsql AS $$ + +BEGIN + ALTER TABLE ts_kv + RENAME TO ts_kv_old; + ALTER TABLE ts_kv_old + RENAME CONSTRAINT ts_kv_pkey TO ts_kv_pkey_old; + CREATE TABLE IF NOT EXISTS ts_kv + ( + LIKE ts_kv_old + ) + PARTITION BY RANGE (ts); + ALTER TABLE ts_kv + DROP COLUMN entity_type; + ALTER TABLE ts_kv + ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid; + ALTER TABLE ts_kv + ALTER COLUMN key TYPE integer USING key::integer; + ALTER TABLE ts_kv + ADD CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts); +END; +$$; + +-- call create_new_ts_kv_latest_table(); + +CREATE OR REPLACE PROCEDURE create_new_ts_kv_latest_table() LANGUAGE plpgsql AS $$ + +BEGIN + ALTER TABLE ts_kv_latest + RENAME TO ts_kv_latest_old; + ALTER TABLE ts_kv_latest_old + RENAME CONSTRAINT ts_kv_latest_pkey TO ts_kv_latest_pkey_old; + CREATE TABLE IF NOT EXISTS ts_kv_latest + ( + LIKE ts_kv_latest_old + ); + ALTER TABLE ts_kv_latest + DROP COLUMN entity_type; + ALTER TABLE ts_kv_latest + ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid; + ALTER TABLE ts_kv_latest + ALTER COLUMN key TYPE integer USING key::integer; + ALTER TABLE ts_kv_latest + ADD CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key); +END; +$$; + +CREATE OR REPLACE FUNCTION get_partitions_data(IN partition_type varchar) + RETURNS + TABLE + ( + partition_date text, + from_ts bigint, + to_ts bigint + ) +AS +$$ +BEGIN + CASE + WHEN partition_type = 'DAYS' THEN + RETURN QUERY SELECT day_date.day AS partition_date, + (extract(epoch from (day_date.day)::timestamp) * 1000)::bigint AS from_ts, + (extract(epoch from (day_date.day::date + INTERVAL '1 DAY')::timestamp) * + 1000)::bigint AS to_ts + FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_MM_DD') AS day + FROM ts_kv_old) AS day_date; + WHEN partition_type = 'MONTHS' THEN + RETURN QUERY SELECT SUBSTRING(month_date.first_date, 1, 7) AS partition_date, + (extract(epoch from (month_date.first_date)::timestamp) * 1000)::bigint AS from_ts, + (extract(epoch from (month_date.first_date::date + INTERVAL '1 MONTH')::timestamp) * + 1000)::bigint AS to_ts + FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_MM_01') AS first_date + FROM ts_kv_old) AS month_date; + WHEN partition_type = 'YEARS' THEN + RETURN QUERY SELECT SUBSTRING(year_date.year, 1, 4) AS partition_date, + (extract(epoch from (year_date.year)::timestamp) * 1000)::bigint AS from_ts, + (extract(epoch from (year_date.year::date + INTERVAL '1 YEAR')::timestamp) * + 1000)::bigint AS to_ts + FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_01_01') AS year FROM ts_kv_old) AS year_date; + ELSE + RAISE EXCEPTION 'Failed to parse partitioning property: % !', partition_type; + END CASE; +END; +$$ LANGUAGE plpgsql; + +-- call create_partitions(); + +CREATE OR REPLACE PROCEDURE create_partitions(IN partition_type varchar) LANGUAGE plpgsql AS $$ + +DECLARE + partition_date varchar; + from_ts bigint; + to_ts bigint; + partitions_cursor CURSOR FOR SELECT * FROM get_partitions_data(partition_type); +BEGIN + OPEN partitions_cursor; + LOOP + FETCH partitions_cursor INTO partition_date, from_ts, to_ts; + EXIT WHEN NOT FOUND; + EXECUTE 'CREATE TABLE IF NOT EXISTS ts_kv_' || partition_date || + ' PARTITION OF ts_kv FOR VALUES FROM (' || from_ts || + ') TO (' || to_ts || ');'; + RAISE NOTICE 'A partition % has been created!',CONCAT('ts_kv_', partition_date); + END LOOP; + + CLOSE partitions_cursor; +END; +$$; + +-- call create_ts_kv_dictionary_table(); + +CREATE OR REPLACE PROCEDURE create_ts_kv_dictionary_table() LANGUAGE plpgsql AS $$ + +BEGIN + CREATE TABLE IF NOT EXISTS ts_kv_dictionary + ( + key varchar(255) NOT NULL, + key_id serial UNIQUE, + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) + ); +END; +$$; + +-- call insert_into_dictionary(); + +CREATE OR REPLACE PROCEDURE insert_into_dictionary() LANGUAGE plpgsql AS $$ + +DECLARE + insert_record RECORD; + key_cursor CURSOR FOR SELECT DISTINCT key + FROM ts_kv_old + ORDER BY key; +BEGIN + OPEN key_cursor; + LOOP + FETCH key_cursor INTO insert_record; + EXIT WHEN NOT FOUND; + IF NOT EXISTS(SELECT key FROM ts_kv_dictionary WHERE key = insert_record.key) THEN + INSERT INTO ts_kv_dictionary(key) VALUES (insert_record.key); + RAISE NOTICE 'Key: % has been inserted into the dictionary!',insert_record.key; + ELSE + RAISE NOTICE 'Key: % already exists in the dictionary!',insert_record.key; + END IF; + END LOOP; + CLOSE key_cursor; +END; +$$; + +-- call insert_into_ts_kv(); + +CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$ +DECLARE + insert_size CONSTANT integer := 10000; + insert_counter integer DEFAULT 0; + insert_record RECORD; + insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id, + ts_kv_records.key AS key, + ts_kv_records.ts AS ts, + ts_kv_records.bool_v AS bool_v, + ts_kv_records.str_v AS str_v, + ts_kv_records.long_v AS long_v, + ts_kv_records.dbl_v AS dbl_v + FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part, + SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part, + SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part, + SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part, + SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part, + key_id AS key, + ts, + bool_v, + str_v, + long_v, + dbl_v + FROM ts_kv_old + INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records; +BEGIN + OPEN insert_cursor; + LOOP + insert_counter := insert_counter + 1; + FETCH insert_cursor INTO insert_record; + IF NOT FOUND THEN + RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter - 1; + EXIT; + END IF; + INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) + VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, + insert_record.long_v, insert_record.dbl_v); + IF MOD(insert_counter, insert_size) = 0 THEN + RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter; + END IF; + END LOOP; + CLOSE insert_cursor; +END; +$$; + +-- call insert_into_ts_kv_latest(); + +CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest() LANGUAGE plpgsql AS $$ +DECLARE + insert_size CONSTANT integer := 10000; + insert_counter integer DEFAULT 0; + insert_record RECORD; + insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id, + ts_kv_latest_records.key AS key, + ts_kv_latest_records.ts AS ts, + ts_kv_latest_records.bool_v AS bool_v, + ts_kv_latest_records.str_v AS str_v, + ts_kv_latest_records.long_v AS long_v, + ts_kv_latest_records.dbl_v AS dbl_v + FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part, + SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part, + SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part, + SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part, + SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part, + key_id AS key, + ts, + bool_v, + str_v, + long_v, + dbl_v + FROM ts_kv_latest_old + INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records; +BEGIN + OPEN insert_cursor; + LOOP + insert_counter := insert_counter + 1; + FETCH insert_cursor INTO insert_record; + IF NOT FOUND THEN + RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter - 1; + EXIT; + END IF; + INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) + VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, + insert_record.long_v, insert_record.dbl_v); + IF MOD(insert_counter, insert_size) = 0 THEN + RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter; + END IF; + END LOOP; + CLOSE insert_cursor; +END; +$$; + + diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql b/application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql new file mode 100644 index 0000000000..982ec9d85f --- /dev/null +++ b/application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql @@ -0,0 +1,179 @@ +-- +-- Copyright © 2016-2020 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- call create_new_ts_kv_table(); + +CREATE OR REPLACE PROCEDURE create_new_ts_kv_table() LANGUAGE plpgsql AS $$ + +BEGIN + ALTER TABLE tenant_ts_kv + RENAME TO tenant_ts_kv_old; + CREATE TABLE IF NOT EXISTS ts_kv + ( + LIKE tenant_ts_kv_old + ); + ALTER TABLE ts_kv ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid; + ALTER TABLE ts_kv ALTER COLUMN key TYPE integer USING key::integer; + ALTER INDEX ts_kv_pkey RENAME TO tenant_ts_kv_pkey_old; + ALTER INDEX idx_tenant_ts_kv RENAME TO idx_tenant_ts_kv_old; + ALTER INDEX tenant_ts_kv_ts_idx RENAME TO tenant_ts_kv_ts_idx_old; + ALTER TABLE ts_kv ADD CONSTRAINT ts_kv_pkey PRIMARY KEY(entity_id, key, ts); +-- CREATE INDEX IF NOT EXISTS ts_kv_ts_idx ON ts_kv(ts DESC); + ALTER TABLE ts_kv DROP COLUMN IF EXISTS tenant_id; +END; +$$; + + +-- call create_ts_kv_latest_table(); + +CREATE OR REPLACE PROCEDURE create_ts_kv_latest_table() LANGUAGE plpgsql AS $$ + +BEGIN + CREATE TABLE IF NOT EXISTS ts_kv_latest + ( + entity_id uuid NOT NULL, + key int NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) + ); +END; +$$; + + +-- call create_ts_kv_dictionary_table(); + +CREATE OR REPLACE PROCEDURE create_ts_kv_dictionary_table() LANGUAGE plpgsql AS $$ + +BEGIN + CREATE TABLE IF NOT EXISTS ts_kv_dictionary + ( + key varchar(255) NOT NULL, + key_id serial UNIQUE, + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) + ); +END; +$$; + +-- call insert_into_dictionary(); + +CREATE OR REPLACE PROCEDURE insert_into_dictionary() LANGUAGE plpgsql AS $$ + +DECLARE + insert_record RECORD; + key_cursor CURSOR FOR SELECT DISTINCT key + FROM tenant_ts_kv_old + ORDER BY key; +BEGIN + OPEN key_cursor; + LOOP + FETCH key_cursor INTO insert_record; + EXIT WHEN NOT FOUND; + IF NOT EXISTS(SELECT key FROM ts_kv_dictionary WHERE key = insert_record.key) THEN + INSERT INTO ts_kv_dictionary(key) VALUES (insert_record.key); + RAISE NOTICE 'Key: % has been inserted into the dictionary!',insert_record.key; + ELSE + RAISE NOTICE 'Key: % already exists in the dictionary!',insert_record.key; + END IF; + END LOOP; + CLOSE key_cursor; +END; +$$; + +-- call insert_into_ts_kv(); + +CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$ + +DECLARE + insert_size CONSTANT integer := 10000; + insert_counter integer DEFAULT 0; + insert_record RECORD; + insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id, + new_ts_kv_records.key AS key, + new_ts_kv_records.ts AS ts, + new_ts_kv_records.bool_v AS bool_v, + new_ts_kv_records.str_v AS str_v, + new_ts_kv_records.long_v AS long_v, + new_ts_kv_records.dbl_v AS dbl_v + FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part, + SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part, + SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part, + SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part, + SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part, + key_id AS key, + ts, + bool_v, + str_v, + long_v, + dbl_v + FROM tenant_ts_kv_old + INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records; +BEGIN + OPEN insert_cursor; + LOOP + insert_counter := insert_counter + 1; + FETCH insert_cursor INTO insert_record; + IF NOT FOUND THEN + RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter - 1; + EXIT; + END IF; + INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) + VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, + insert_record.long_v, insert_record.dbl_v); + IF MOD(insert_counter, insert_size) = 0 THEN + RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter; + END IF; + END LOOP; + CLOSE insert_cursor; +END; +$$; + +-- call insert_into_ts_kv_latest(); + +CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest() LANGUAGE plpgsql AS $$ + +DECLARE + insert_size CONSTANT integer := 10000; + insert_counter integer DEFAULT 0; + latest_record RECORD; + insert_record RECORD; + insert_cursor CURSOR FOR SELECT + latest_records.key AS key, + latest_records.entity_id AS entity_id, + latest_records.ts AS ts + FROM (SELECT DISTINCT key AS key, entity_id AS entity_id, MAX(ts) AS ts FROM ts_kv GROUP BY key, entity_id) AS latest_records; +BEGIN + OPEN insert_cursor; + LOOP + insert_counter := insert_counter + 1; + FETCH insert_cursor INTO latest_record; + IF NOT FOUND THEN + RAISE NOTICE '% records have been inserted into the ts_kv_latest table!',insert_counter - 1; + EXIT; + END IF; + SELECT entity_id AS entity_id, key AS key, ts AS ts, bool_v AS bool_v, str_v AS str_v, long_v AS long_v, dbl_v AS dbl_v INTO insert_record FROM ts_kv WHERE entity_id = latest_record.entity_id AND key = latest_record.key AND ts = latest_record.ts; + INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) + VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, insert_record.long_v, insert_record.dbl_v); + IF MOD(insert_counter, insert_size) = 0 THEN + RAISE NOTICE '% records have been inserted into the ts_kv_latest table!',insert_counter; + END IF; + END LOOP; + CLOSE insert_cursor; +END; +$$; diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql new file mode 100644 index 0000000000..dda3bd7b1c --- /dev/null +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -0,0 +1,150 @@ +-- +-- Copyright © 2016-2020 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS +$$ +BEGIN + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(device.id) as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(asset.id) as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(customer.id) as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid varchar(31), + IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + tenant_cursor CURSOR FOR select tenant.id as tenant_id + from tenant; + tenant_id_record varchar; + customer_id_record varchar; + tenant_ttl bigint; + customer_ttl bigint; + deleted_for_entities bigint; + tenant_ttl_ts bigint; + customer_ttl_ts bigint; +BEGIN + OPEN tenant_cursor; + FETCH tenant_cursor INTO tenant_id_record; + WHILE FOUND + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + tenant_id_record, 'TTL') INTO tenant_ttl; + if tenant_ttl IS NULL THEN + tenant_ttl := system_ttl; + END IF; + IF tenant_ttl > 0 THEN + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; + END IF; + FOR customer_id_record IN + SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + customer_id_record, 'TTL') INTO customer_ttl; + IF customer_ttl IS NULL THEN + customer_ttl_ts := tenant_ttl_ts; + ELSE + IF customer_ttl > 0 THEN + customer_ttl_ts := + (EXTRACT(EPOCH FROM current_timestamp) * 1000 - + customer_ttl::bigint * 1000)::bigint; + END IF; + END IF; + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN + deleted_for_entities := + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; + deleted_for_entities := + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, + customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + END IF; + END LOOP; + FETCH tenant_cursor INTO tenant_id_record; + END LOOP; +END +$$; + +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; +BEGIN + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; + END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; + END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; +END +$$; diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index cd51e9c6bc..28f5b188e5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import lombok.Getter; @@ -36,7 +37,6 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.common.data.DataConstants; @@ -44,10 +44,11 @@ import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -64,24 +65,23 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.kafka.TbNodeIdProvider; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; import org.thingsboard.server.service.mail.MailExecutorService; -import org.thingsboard.server.service.rpc.DeviceRpcService; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; +import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.script.JsExecutorService; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.session.DeviceSessionCacheService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.service.transport.RuleEngineTransportService; +import org.thingsboard.server.service.transport.TbCoreToTransportService; import javax.annotation.Nullable; import java.io.IOException; @@ -105,35 +105,24 @@ public class ActorSystemContext { return debugPerTenantLimits; } + @Autowired @Getter @Setter - private ActorService actorService; + private TbServiceInfoProvider serviceInfoProvider; - @Autowired @Getter - private DiscoveryService discoveryService; + @Setter + private ActorService actorService; @Autowired @Getter @Setter private ComponentDiscoveryService componentService; - @Autowired - @Getter - private ClusterRoutingService routingService; - - @Autowired - @Getter - private ClusterRpcService rpcService; - @Autowired @Getter private DataDecodingEncodingService encodingService; - @Autowired - @Getter - private DeviceAuthService deviceAuthService; - @Autowired @Getter private DeviceService deviceService; @@ -162,6 +151,13 @@ public class ActorSystemContext { @Getter private RuleChainService ruleChainService; + @Autowired + private PartitionService partitionService; + + @Autowired + @Getter + private TbClusterService clusterService; + @Autowired @Getter private TimeseriesService tsService; @@ -194,10 +190,6 @@ public class ActorSystemContext { @Getter private TelemetrySubscriptionService tsSubService; - @Autowired - @Getter - private DeviceRpcService deviceRpcService; - @Autowired @Getter private JsInvokeService jsSandbox; @@ -210,10 +202,6 @@ public class ActorSystemContext { @Getter private MailExecutorService mailExecutor; - @Autowired - @Getter - private ClusterRpcCallbackExecutorService clusterRpcCallbackExecutor; - @Autowired @Getter private DbCallbackExecutorService dbCallbackExecutor; @@ -230,27 +218,34 @@ public class ActorSystemContext { @Getter private MailService mailService; - @Autowired + //TODO: separate context for TbCore and TbRuleEngine + @Autowired(required = false) @Getter private DeviceStateService deviceStateService; - @Autowired + @Autowired(required = false) @Getter private DeviceSessionCacheService deviceSessionCacheService; - @Lazy - @Autowired + @Autowired(required = false) @Getter - private RuleEngineTransportService ruleEngineTransportService; + private TbCoreToTransportService tbCoreToTransportService; + /** + * The following Service will be null if we operate in tb-core mode + */ @Lazy - @Autowired + @Autowired(required = false) @Getter - private RuleChainTransactionService ruleChainTransactionService; + private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; - @Value("${cluster.partition_id}") + /** + * The following Service will be null if we operate in tb-rule-engine mode + */ + @Lazy + @Autowired(required = false) @Getter - private long queuePartitionId; + private TbCoreDeviceRpcService tbCoreDeviceRpcService; @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter @@ -268,10 +263,6 @@ public class ActorSystemContext { @Getter private long queuePersistenceTimeout; - @Value("${actors.client_side_rpc.timeout}") - @Getter - private long clientSideRpcTimeout; - @Value("${actors.rule.chain.error_persist_frequency}") @Getter private long ruleChainErrorPersistFrequency; @@ -295,7 +286,7 @@ public class ActorSystemContext { @Getter private final AtomicInteger jsInvokeFailuresCount = new AtomicInteger(0); - @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}") + @Scheduled(fixedDelayString = "${actors.statistics.js_print_interval_ms}") public void printStats() { if (statisticsEnabled) { if (jsInvokeRequestsCount.get() > 0 || jsInvokeResponsesCount.get() > 0 || jsInvokeFailuresCount.get() > 0) { @@ -333,11 +324,6 @@ public class ActorSystemContext { @Setter private ActorSystem actorSystem; - @Autowired - @Getter - private TbNodeIdProvider nodeIdProvider; - - @Getter @Setter private ActorRef appActor; @@ -364,6 +350,8 @@ public class ActorSystemContext { config = ConfigFactory.parseResources(AKKA_CONF_FILE_NAME).withFallback(ConfigFactory.load()); } + + public Scheduler getScheduler() { return actorSystem.scheduler(); } @@ -373,7 +361,7 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.ERROR); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); + event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e))); persistEvent(event); } @@ -382,7 +370,7 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); + event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -396,8 +384,8 @@ public class ActorSystemContext { return sw.toString(); } - private JsonNode toBodyJson(ServerAddress server, ComponentLifecycleEvent event, Optional e) { - ObjectNode node = mapper.createObjectNode().put("server", server.toString()).put("event", event.name()); + private JsonNode toBodyJson(String serviceId, ComponentLifecycleEvent event, Optional e) { + ObjectNode node = mapper.createObjectNode().put("server", serviceId).put("event", event.name()); if (e.isPresent()) { node = node.put("success", false); node = node.put("error", toString(e.get())); @@ -407,12 +395,21 @@ public class ActorSystemContext { return node; } - private JsonNode toBodyJson(ServerAddress server, String method, String body) { - return mapper.createObjectNode().put("server", server.toString()).put("method", method).put("error", body); + private JsonNode toBodyJson(String serviceId, String method, String body) { + return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); } - public String getServerAddress() { - return discoveryService.getCurrentServer().getServerAddress().toString(); + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + return partitionService.resolve(serviceType, tenantId, entityId); + } + + public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { + return partitionService.resolve(serviceType, queueName, tenantId, entityId); + } + + + public String getServiceId() { + return serviceInfoProvider.getServiceId(); } public void persistDebugInput(TenantId tenantId, EntityId entityId, TbMsg tbMsg, String relationType) { @@ -443,7 +440,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() .put("type", type) - .put("server", getServerAddress()) + .put("server", getServiceId()) .put("entityId", tbMsg.getOriginator().getId().toString()) .put("entityName", tbMsg.getOriginator().getEntityType().name()) .put("msgId", tbMsg.getId().toString()) @@ -469,7 +466,7 @@ public class ActorSystemContext { public void onFailure(Throwable th) { log.error("Could not save debug Event for Node", th); } - }); + }, MoreExecutors.directExecutor()); } catch (IOException ex) { log.warn("Failed to persist rule node debug message", ex); } @@ -503,7 +500,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() //todo: what fields are needed here? - .put("server", getServerAddress()) + .put("server", getServiceId()) .put("message", "Reached debug mode rate limit!"); if (error != null) { @@ -522,11 +519,14 @@ public class ActorSystemContext { public void onFailure(Throwable th) { log.error("Could not save debug Event for Rule Chain", th); } - }); + }, MoreExecutors.directExecutor()); } public static Exception toException(Throwable error) { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } + public void tell(TbActorMsg tbActorMsg, ActorRef sender) { + appActor.tell(tbActorMsg, sender); + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 778c945a32..b7782a6a87 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -20,19 +20,13 @@ import akka.actor.LocalActorRef; import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; -import akka.actor.SupervisorStrategy.Directive; import akka.actor.Terminated; -import akka.event.Logging; -import akka.event.LoggingAdapter; -import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; +import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.actors.shared.rulechain.SystemRuleChainManager; import org.thingsboard.server.actors.tenant.TenantActor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; @@ -42,29 +36,32 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import scala.concurrent.duration.Duration; -import java.util.HashMap; -import java.util.Map; +import java.util.HashSet; import java.util.Optional; +import java.util.Set; -public class AppActor extends RuleChainManagerActor { +public class AppActor extends ContextAwareActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); private final TenantService tenantService; private final BiMap tenantActors; + private final Set deletedTenants; private boolean ruleChainsInitialized; private AppActor(ActorSystemContext systemContext) { - super(systemContext, new SystemRuleChainManager(systemContext)); + super(systemContext); this.tenantService = systemContext.getTenantService(); this.tenantActors = HashBiMap.create(); + this.deletedTenants = new HashSet<>(); } @Override @@ -79,7 +76,7 @@ public class AppActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { if (!ruleChainsInitialized) { - initRuleChainsAndTenantActors(); + initTenantActors(); ruleChainsInitialized = true; if (msg.getMsgType() != MsgType.APP_INIT_MSG) { log.warn("Rule Chains initialized by unexpected message: {}", msg); @@ -88,17 +85,14 @@ public class AppActor extends RuleChainManagerActor { switch (msg.getMsgType()) { case APP_INIT_MSG: break; - case SEND_TO_CLUSTER_MSG: - onPossibleClusterMsg((SendToClusterMsg) msg); - break; - case CLUSTER_EVENT_MSG: + case PARTITION_CHANGE_MSG: broadcast(msg); break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case SERVICE_TO_RULE_ENGINE_MSG: - onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); + case QUEUE_TO_RULE_ENGINE_MSG: + onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -106,7 +100,6 @@ public class AppActor extends RuleChainManagerActor { case DEVICE_NAME_OR_TYPE_UPDATE_TO_DEVICE_ACTOR_MSG: case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: - case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onToDeviceActorMsg((TenantAwareMsg) msg); break; default: @@ -115,16 +108,30 @@ public class AppActor extends RuleChainManagerActor { return true; } - private void initRuleChainsAndTenantActors() { + private void initTenantActors() { log.info("Starting main system actor."); try { - initRuleChains(); - if (systemContext.isTenantComponentsInitEnabled()) { - PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); - for (Tenant tenant : tenantIterator) { + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + if (isolatedTenantId.isPresent()) { + Tenant tenant = systemContext.getTenantService().findTenantById(isolatedTenantId.get()); + if (tenant != null) { log.debug("[{}] Creating tenant actor", tenant.getId()); getOrCreateTenantActor(tenant.getId()); log.debug("Tenant actor created."); + } else { + log.error("[{}] Tenant with such ID does not exist", isolatedTenantId.get()); + } + } else if (systemContext.isTenantComponentsInitEnabled()) { + PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); + boolean isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + boolean isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + for (Tenant tenant : tenantIterator) { + if (isCore || (isRuleEngine && !tenant.isIsolatedTbRuleEngine())) { + log.debug("[{}] Creating tenant actor", tenant.getId()); + getOrCreateTenantActor(tenant.getId()); + log.debug("[{}] Tenant actor created.", tenant.getId()); + } } } log.info("Main system actor started."); @@ -133,40 +140,33 @@ public class AppActor extends RuleChainManagerActor { } } - private void onPossibleClusterMsg(SendToClusterMsg msg) { - Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); - if (address.isPresent()) { - systemContext.getRpcService().tell( - systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); - } else { - self().tell(msg.getMsg(), ActorRef.noSender()); - } - } - - private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { + private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { if (SYSTEM_TENANT.equals(msg.getTenantId())) { -// this may be a notification about system entities created. -// log.warn("[{}] Invalid service to rule engine msg called. System messages are not supported yet: {}", SYSTEM_TENANT, msg); + msg.getTbMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); + if (!deletedTenants.contains(msg.getTenantId())) { + getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); + } else { + msg.getTbMsg().getCallback().onSuccess(); + } } } - @Override protected void broadcast(Object msg) { - super.broadcast(msg); tenantActors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { ActorRef target = null; if (SYSTEM_TENANT.equals(msg.getTenantId())) { - target = getEntityActorRef(msg.getEntityId()); + log.warn("Message has system tenant id: {}", msg); } else { if (msg.getEntityId().getEntityType() == EntityType.TENANT && msg.getEvent() == ComponentLifecycleEvent.DELETED) { - log.debug("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); - ActorRef tenantActor = tenantActors.remove(new TenantId(msg.getEntityId().getId())); + log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); + TenantId tenantId = new TenantId(msg.getEntityId().getId()); + deletedTenants.add(tenantId); + ActorRef tenantActor = tenantActors.get(tenantId); if (tenantActor != null) { log.debug("[{}] Deleting tenant actor: {}", msg.getTenantId(), tenantActor); context().stop(tenantActor); @@ -183,16 +183,22 @@ public class AppActor extends RuleChainManagerActor { } private void onToDeviceActorMsg(TenantAwareMsg msg) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); + if (!deletedTenants.contains(msg.getTenantId())) { + getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); + } else { + if (msg instanceof TransportToDeviceActorMsgWrapper) { + ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); + } + } } private ActorRef getOrCreateTenantActor(TenantId tenantId) { return tenantActors.computeIfAbsent(tenantId, k -> { - log.debug("[{}] Creating tenant actor.", tenantId); + log.info("[{}] Creating tenant actor.", tenantId); ActorRef tenantActor = context().actorOf(Props.create(new TenantActor.ActorCreator(systemContext, tenantId)) .withDispatcher(DefaultActorService.CORE_DISPATCHER_NAME), tenantId.toString()); context().watch(tenantActor); - log.debug("[{}] Created tenant actor: {}.", tenantId, tenantActor); + log.info("[{}] Created tenant actor: {}.", tenantId, tenantActor); return tenantActor; }); } diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java index 547e484652..256ffd0b30 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java @@ -22,10 +22,8 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; public class DeviceActor extends ContextAwareActor { @@ -48,6 +46,11 @@ public class DeviceActor extends ContextAwareActor { } } + @Override + public void postStop() { + + } + @Override protected boolean process(TbActorMsg msg) { switch (msg.getMsgType()) { @@ -66,15 +69,9 @@ public class DeviceActor extends ContextAwareActor { case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: processor.processRpcRequest(context(), (ToDeviceRpcRequestActorMsg) msg); break; - case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: - processor.processToServerRPCResponse(context(), (ToServerRpcResponseActorMsg) msg); - break; case DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG: processor.processServerSideRpcTimeout(context(), (DeviceActorServerSideRpcTimeoutMsg) msg); break; - case DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG: - processor.processClientSideRpcTimeout(context(), (DeviceActorClientSideRpcTimeoutMsg) msg); - break; case SESSION_TIMEOUT_MSG: processor.checkSessionsTimeout(); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 7eb91847cf..437ed456e9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -16,13 +16,10 @@ package org.thingsboard.server.actors.device; import akka.actor.ActorContext; -import com.datastax.driver.core.utils.UUIDs; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; +import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -38,36 +35,33 @@ import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.session.SessionMsgType; -import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SessionSubscriptionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SessionType; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.Nullable; @@ -99,10 +93,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private final Map attributeSubscriptions; private final Map rpcSubscriptions; private final Map toDeviceRpcPendingMap; - private final Map toServerRpcPendingMap; - - private final Gson gson = new Gson(); - private final JsonParser jsonParser = new JsonParser(); private int rpcSeq = 0; private String deviceName; @@ -117,7 +107,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { this.attributeSubscriptions = new HashMap<>(); this.rpcSubscriptions = new HashMap<>(); this.toDeviceRpcPendingMap = new HashMap<>(); - this.toServerRpcPendingMap = new HashMap<>(); if (initAttributes()) { restoreSessions(); } @@ -153,7 +142,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { Set syncSessionSet = new HashSet<>(); rpcSubscriptions.forEach((key, value) -> { sendToTransport(rpcRequest, key, value.getNodeId()); - if (TransportProtos.SessionType.SYNC == value.getType()) { + if (SessionType.SYNC == value.getType()) { syncSessionSet.add(key); } }); @@ -161,7 +150,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (request.isOneway() && sent) { log.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); } else { registerPendingRpcRequest(context, msg, sent, rpcRequest, timeout); } @@ -182,16 +171,16 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(msg.getId()); if (requestMd != null) { log.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION)); } } private void sendPendingRequests(ActorContext context, UUID sessionId, SessionInfoProto sessionInfo) { - TransportProtos.SessionType sessionType = getSessionType(sessionId); + SessionType sessionType = getSessionType(sessionId); if (!toDeviceRpcPendingMap.isEmpty()) { log.debug("[{}] Pushing {} pending RPC messages to new async session [{}]", deviceId, toDeviceRpcPendingMap.size(), sessionId); - if (sessionType == TransportProtos.SessionType.SYNC) { + if (sessionType == SessionType.SYNC) { log.debug("[{}] Cleanup sync rpc session [{}]", deviceId, sessionId); rpcSubscriptions.remove(sessionId); } @@ -199,7 +188,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No pending RPC messages for new async session [{}]", deviceId, sessionId); } Set sentOneWayIds = new HashSet<>(); - if (sessionType == TransportProtos.SessionType.ASYNC) { + if (sessionType == SessionType.ASYNC) { toDeviceRpcPendingMap.entrySet().forEach(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); } else { toDeviceRpcPendingMap.entrySet().stream().findFirst().ifPresent(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); @@ -214,7 +203,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestBody body = request.getBody(); if (request.isOneway()) { sentOneWayIds.add(entry.getKey()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); } ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder().setRequestId( entry.getKey()).setMethodName(body.getMethod()).setParams(body.getParams()).build(); @@ -223,8 +212,8 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { - boolean reportDeviceActivity = false; TransportToDeviceActorMsg msg = wrapper.getMsg(); + TbCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { processSessionStateMsgs(msg.getSessionInfo(), msg.getSessionEvent()); } @@ -234,34 +223,16 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (msg.hasSubscribeToRPC()) { processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToRPC()); } - if (msg.hasPostAttributes()) { - handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes()); - reportDeviceActivity = true; - } - if (msg.hasPostTelemetry()) { - handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry()); - reportDeviceActivity = true; - } if (msg.hasGetAttributes()) { handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes()); } if (msg.hasToDeviceRPCCallResponse()) { processRpcResponses(context, msg.getSessionInfo(), msg.getToDeviceRPCCallResponse()); } - if (msg.hasToServerRPCCallRequest()) { - handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest()); - reportDeviceActivity = true; - } if (msg.hasSubscriptionInfo()) { handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo()); } - if (reportDeviceActivity) { - reportLogicalDeviceActivity(); - } - } - - private void reportLogicalDeviceActivity() { - systemContext.getDeviceStateService().onDeviceActivity(deviceId); + callback.onSuccess(); } private void reportSessionOpen() { @@ -292,7 +263,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .build(); sendToTransport(responseMsg, sessionInfo); } - }); + }, MoreExecutors.directExecutor()); } private ListenableFuture>> getAttributesKvEntries(GetAttributeRequestMsg request) { @@ -326,67 +297,8 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { return new HashSet<>(strings); } - private void handlePostAttributesRequest(ActorContext context, SessionInfoProto sessionInfo, PostAttributeMsg postAttributes) { - JsonObject json = getJsonObject(postAttributes.getKvList()); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, defaultMetaData.copy(), - TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - pushToRuleEngine(context, tbMsg); - } - - private void handlePostTelemetryRequest(ActorContext context, SessionInfoProto sessionInfo, PostTelemetryMsg postTelemetry) { - for (TsKvListProto tsKv : postTelemetry.getTsKvListList()) { - JsonObject json = getJsonObject(tsKv.getKvList()); - TbMsgMetaData metaData = defaultMetaData.copy(); - metaData.putValue("ts", tsKv.getTs() + ""); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - pushToRuleEngine(context, tbMsg); - } - } - - private void handleClientSideRPCRequest(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg request) { - UUID sessionId = getSessionId(sessionInfo); - JsonObject json = new JsonObject(); - json.addProperty("method", request.getMethodName()); - json.add("params", jsonParser.parse(request.getParams())); - - TbMsgMetaData requestMetaData = defaultMetaData.copy(); - requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); - - scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); - toServerRpcPendingMap.put(request.getRequestId(), new ToServerRpcRequestMetadata(sessionId, getSessionType(sessionId), sessionInfo.getNodeId())); - } - - private TransportProtos.SessionType getSessionType(UUID sessionId) { - return sessions.containsKey(sessionId) ? TransportProtos.SessionType.ASYNC : TransportProtos.SessionType.SYNC; - } - - void processClientSideRpcTimeout(ActorContext context, DeviceActorClientSideRpcTimeoutMsg msg) { - ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(msg.getId()); - if (data != null) { - log.debug("[{}] Client side RPC request [{}] timeout detected!", deviceId, msg.getId()); - sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(msg.getId()).setError("timeout").build() - , data.getSessionId(), data.getNodeId()); - } - } - - void processToServerRPCResponse(ActorContext context, ToServerRpcResponseActorMsg msg) { - int requestId = msg.getMsg().getRequestId(); - ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); - if (data != null) { - log.debug("[{}] Pushing reply to [{}][{}]!", deviceId, data.getNodeId(), data.getSessionId()); - sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(requestId).setPayload(msg.getMsg().getData()).build() - , data.getSessionId(), data.getNodeId()); - } else { - log.debug("[{}][{}] Pending RPC request to server not found!", deviceId, requestId); - } - } - - private void pushToRuleEngine(ActorContext context, TbMsg tbMsg) { - context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); + private SessionType getSessionType(UUID sessionId) { + return sessions.containsKey(sessionId) ? SessionType.ASYNC : SessionType.SYNC; } void processAttributesUpdate(ActorContext context, DeviceAttributesEventNotificationMsg msg) { @@ -434,7 +346,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(responseMsg.getRequestId()); boolean success = requestMd != null; if (success) { - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), responseMsg.getPayload(), null)); } else { log.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId()); @@ -449,7 +361,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToAttributes(true); log.debug("[{}] Registering attributes subscription for session [{}]", deviceId, sessionId); @@ -470,7 +382,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToRPC(true); log.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId); @@ -494,10 +406,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { notifyTransportAboutClosedSession(sessionIdToRemove, sessions.remove(sessionIdToRemove)); } } - sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfo.getNodeId()))); + sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfo.getNodeId()))); if (sessions.size() == 1) { reportSessionOpen(); } + systemContext.getDeviceStateService().onDeviceActivity(deviceId, System.currentTimeMillis()); dumpSessions(); } else if (msg.getEvent() == SessionEvent.CLOSED) { log.debug("[{}] Canceling subscriptions for closed session [{}]", deviceId, sessionId); @@ -511,10 +424,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, TransportProtos.SubscriptionInfoProto subscriptionInfo) { + private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, SubscriptionInfoProto subscriptionInfo) { UUID sessionId = getSessionId(sessionInfoProto); SessionInfoMetaData sessionMD = sessions.computeIfAbsent(sessionId, - id -> new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); + id -> new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); sessionMD.setLastActivityTime(subscriptionInfo.getLastActivityTime()); sessionMD.setSubscribedToAttributes(subscriptionInfo.getAttributeSubscription()); @@ -525,6 +438,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (subscriptionInfo.getRpcSubscription()) { rpcSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo()); } + systemContext.getDeviceStateService().onDeviceActivity(deviceId, subscriptionInfo.getLastActivityTime()); dumpSessions(); } @@ -536,11 +450,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void notifyTransportAboutClosedSession(UUID sessionId, SessionInfoMetaData sessionMd) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setSessionCloseNotification(SessionCloseNotificationProto.getDefaultInstance()).build(); - systemContext.getRuleEngineTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); + systemContext.getTbCoreToTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); } void processNameOrTypeUpdate(DeviceNameOrTypeUpdateMsg msg) { @@ -551,57 +465,36 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { this.defaultMetaData.putValue("deviceType", deviceType); } - private JsonObject getJsonObject(List tsKv) { - JsonObject json = new JsonObject(); - for (KeyValueProto kv : tsKv) { - switch (kv.getType()) { - case BOOLEAN_V: - json.addProperty(kv.getKey(), kv.getBoolV()); - break; - case LONG_V: - json.addProperty(kv.getKey(), kv.getLongV()); - break; - case DOUBLE_V: - json.addProperty(kv.getKey(), kv.getDoubleV()); - break; - case STRING_V: - json.addProperty(kv.getKey(), kv.getStringV()); - break; - } - } - return json; - } - private void sendToTransport(GetAttributeResponseMsg responseMsg, SessionInfoProto sessionInfo) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionInfo.getSessionIdMSB()) .setSessionIdLSB(sessionInfo.getSessionIdLSB()) .setGetAttributesResponse(responseMsg).build(); - systemContext.getRuleEngineTransportService().process(sessionInfo.getNodeId(), msg); + systemContext.getTbCoreToTransportService().process(sessionInfo.getNodeId(), msg); } private void sendToTransport(AttributeUpdateNotificationMsg notificationMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setAttributeUpdateNotification(notificationMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } private void sendToTransport(ToDeviceRpcRequestMsg rpcMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToDeviceRequest(rpcMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } - private void sendToTransport(TransportProtos.ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + private void sendToTransport(ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToServerResponse(rpcMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } @@ -643,15 +536,19 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { builder.setType(KeyValueType.STRING_V); builder.setStringV(kvEntry.getStrValue().get()); break; + case JSON: + builder.setType(KeyValueType.JSON_V); + builder.setJsonV(kvEntry.getJsonValue().get()); + break; } return builder.build(); } private void restoreSessions() { log.debug("[{}] Restoring sessions from cache", deviceId); - TransportProtos.DeviceSessionsCacheEntry sessionsDump = null; + DeviceSessionsCacheEntry sessionsDump = null; try { - sessionsDump = TransportProtos.DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); + sessionsDump = DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); } catch (InvalidProtocolBufferException e) { log.warn("[{}] Failed to decode device sessions from cache", deviceId); return; @@ -660,11 +557,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No session information found", deviceId); return; } - for (TransportProtos.SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { + for (SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { SessionInfoProto sessionInfoProto = sessionSubscriptionInfoProto.getSessionInfo(); UUID sessionId = getSessionId(sessionInfoProto); - SessionInfo sessionInfo = new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()); - TransportProtos.SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); + SessionInfo sessionInfo = new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()); + SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); SessionInfoMetaData sessionMD = new SessionInfoMetaData(sessionInfo, subInfo.getLastActivityTime()); sessions.put(sessionId, sessionMD); if (subInfo.getAttributeSubscription()) { @@ -682,27 +579,27 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void dumpSessions() { log.debug("[{}] Dumping sessions: {}, rpc subscriptions: {}, attribute subscriptions: {} to cache", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size()); - List sessionsList = new ArrayList<>(sessions.size()); + List sessionsList = new ArrayList<>(sessions.size()); sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getSessionInfo().getType() == TransportProtos.SessionType.SYNC) { + if (sessionMD.getSessionInfo().getType() == SessionType.SYNC) { return; } SessionInfo sessionInfo = sessionMD.getSessionInfo(); - TransportProtos.SubscriptionInfoProto subscriptionInfoProto = TransportProtos.SubscriptionInfoProto.newBuilder() + SubscriptionInfoProto subscriptionInfoProto = SubscriptionInfoProto.newBuilder() .setLastActivityTime(sessionMD.getLastActivityTime()) .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) .setRpcSubscription(sessionMD.isSubscribedToRPC()).build(); - TransportProtos.SessionInfoProto sessionInfoProto = TransportProtos.SessionInfoProto.newBuilder() + SessionInfoProto sessionInfoProto = SessionInfoProto.newBuilder() .setSessionIdMSB(uuid.getMostSignificantBits()) .setSessionIdLSB(uuid.getLeastSignificantBits()) .setNodeId(sessionInfo.getNodeId()).build(); - sessionsList.add(TransportProtos.SessionSubscriptionInfoProto.newBuilder() + sessionsList.add(SessionSubscriptionInfoProto.newBuilder() .setSessionInfo(sessionInfoProto) .setSubscriptionInfo(subscriptionInfoProto).build()); log.debug("[{}] Dumping session: {}", deviceId, sessionMD); }); systemContext.getDeviceSessionCacheService() - .put(deviceId, TransportProtos.DeviceSessionsCacheEntry.newBuilder() + .put(deviceId, DeviceSessionsCacheEntry.newBuilder() .addAllSessions(sessionsList).build().toByteArray()); } @@ -723,4 +620,5 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { dumpSessions(); } } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java b/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java deleted file mode 100644 index 8b502b973c..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; -import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class BasicRpcSessionListener implements GrpcSessionListener { - - private final ClusterRpcCallbackExecutorService callbackExecutorService; - private final ActorService service; - private final ActorRef manager; - private final ActorRef self; - - BasicRpcSessionListener(ActorSystemContext context, ActorRef manager, ActorRef self) { - this.service = context.getActorService(); - this.callbackExecutorService = context.getClusterRpcCallbackExecutor(); - this.manager = manager; - this.self = self; - } - - @Override - public void onConnected(GrpcSession session) { - log.info("[{}][{}] session started", session.getRemoteServer(), getType(session)); - if (!session.isClient()) { - manager.tell(new RpcSessionConnectedMsg(session.getRemoteServer(), session.getSessionId()), self); - } - } - - @Override - public void onDisconnected(GrpcSession session) { - log.info("[{}][{}] session closed", session.getRemoteServer(), getType(session)); - manager.tell(new RpcSessionDisconnectedMsg(session.isClient(), session.getRemoteServer()), self); - } - - @Override - public void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage) { - log.trace("Received session actor msg from [{}][{}]: {}", session.getRemoteServer(), getType(session), clusterMessage); - callbackExecutorService.execute(() -> { - try { - service.onReceivedMsg(session.getRemoteServer(), clusterMessage); - } catch (Exception e) { - log.debug("[{}][{}] Failed to process cluster message: {}", session.getRemoteServer(), getType(session), clusterMessage, e); - } - }); - } - - @Override - public void onError(GrpcSession session, Throwable t) { - log.warn("[{}][{}] session got error -> {}", session.getRemoteServer(), getType(session), t); - manager.tell(new RpcSessionClosedMsg(session.isClient(), session.getRemoteServer()), self); - session.close(); - } - - private static String getType(GrpcSession session) { - return session.isClient() ? "Client" : "Server"; - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java deleted file mode 100644 index 133e7375d4..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java +++ /dev/null @@ -1,230 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import akka.actor.OneForOneStrategy; -import akka.actor.Props; -import akka.actor.SupervisorStrategy; -import akka.event.Logging; -import akka.event.LoggingAdapter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import scala.concurrent.duration.Duration; - -import java.util.*; - -/** - * @author Andrew Shvayka - */ -public class RpcManagerActor extends ContextAwareActor { - - private final Map sessionActors; - private final Map> pendingMsgs; - private final ServerAddress instance; - - private RpcManagerActor(ActorSystemContext systemContext) { - super(systemContext); - this.sessionActors = new HashMap<>(); - this.pendingMsgs = new HashMap<>(); - this.instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - - systemContext.getDiscoveryService().getOtherServers().stream() - .filter(otherServer -> otherServer.getServerAddress().compareTo(instance) > 0) - .forEach(otherServer -> onCreateSessionRequest( - new RpcSessionCreateRequestMsg(UUID.randomUUID(), otherServer.getServerAddress(), null))); - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - onMsg((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcBroadcastMsg) { - onMsg((RpcBroadcastMsg) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - onCreateSessionRequest((RpcSessionCreateRequestMsg) msg); - } else if (msg instanceof RpcSessionConnectedMsg) { - onSessionConnected((RpcSessionConnectedMsg) msg); - } else if (msg instanceof RpcSessionDisconnectedMsg) { - onSessionDisconnected((RpcSessionDisconnectedMsg) msg); - } else if (msg instanceof RpcSessionClosedMsg) { - onSessionClosed((RpcSessionClosedMsg) msg); - } else if (msg instanceof ClusterEventMsg) { - onClusterEvent((ClusterEventMsg) msg); - } - } - - private void onMsg(RpcBroadcastMsg msg) { - log.debug("Forwarding msg to session actors {}", msg); - sessionActors.keySet().forEach(address -> { - ClusterAPIProtos.ClusterMessage msgWithServerAddress = msg.getMsg() - .toBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(address.getHost()) - .setPort(address.getPort()) - .build()) - .build(); - onMsg(msgWithServerAddress); - }); - pendingMsgs.values().forEach(queue -> queue.add(msg.getMsg())); - } - - private void onMsg(ClusterAPIProtos.ClusterMessage msg) { - if (msg.hasServerAddress()) { - ServerAddress address = new ServerAddress(msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE); - SessionActorInfo session = sessionActors.get(address); - if (session != null) { - log.debug("{} Forwarding msg to session actor: {}", address, msg); - session.getActor().tell(msg, ActorRef.noSender()); - } else { - log.debug("{} Storing msg to pending queue: {}", address, msg); - Queue queue = pendingMsgs.get(address); - if (queue == null) { - queue = new LinkedList<>(); - pendingMsgs.put(new ServerAddress( - msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE), queue); - } - queue.add(msg); - } - } else { - log.warn("Cluster msg doesn't have server address [{}]", msg); - } - } - - @Override - public void postStop() { - sessionActors.clear(); - pendingMsgs.clear(); - } - - private void onClusterEvent(ClusterEventMsg msg) { - ServerAddress server = msg.getServerAddress(); - if (server.compareTo(instance) > 0) { - if (msg.isAdded()) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(UUID.randomUUID(), server, null)); - } else { - onSessionClose(false, server); - } - } - } - - private void onSessionConnected(RpcSessionConnectedMsg msg) { - register(msg.getRemoteAddress(), msg.getId(), context().sender()); - } - - private void onSessionDisconnected(RpcSessionDisconnectedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private void onSessionClosed(RpcSessionClosedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private boolean isRegistered(ServerAddress address) { - for (ServerInstance server : systemContext.getDiscoveryService().getOtherServers()) { - if (server.getServerAddress().equals(address)) { - return true; - } - } - return false; - } - - private void onSessionClose(boolean reconnect, ServerAddress remoteAddress) { - log.info("[{}] session closed. Should reconnect: {}", remoteAddress, reconnect); - SessionActorInfo sessionRef = sessionActors.get(remoteAddress); - if (sessionRef != null && context().sender() != null && context().sender().equals(sessionRef.actor)) { - context().stop(sessionRef.actor); - sessionActors.remove(remoteAddress); - pendingMsgs.remove(remoteAddress); - if (reconnect) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(sessionRef.sessionId, remoteAddress, null)); - } - } - } - - private void onCreateSessionRequest(RpcSessionCreateRequestMsg msg) { - if (msg.getRemoteAddress() != null) { - if (!sessionActors.containsKey(msg.getRemoteAddress())) { - ActorRef actorRef = createSessionActor(msg); - register(msg.getRemoteAddress(), msg.getMsgUid(), actorRef); - } - } else { - createSessionActor(msg); - } - } - - private void register(ServerAddress remoteAddress, UUID uuid, ActorRef sender) { - sessionActors.put(remoteAddress, new SessionActorInfo(uuid, sender)); - log.info("[{}][{}] Registering session actor.", remoteAddress, uuid); - Queue data = pendingMsgs.remove(remoteAddress); - if (data != null) { - log.info("[{}][{}] Forwarding {} pending messages.", remoteAddress, uuid, data.size()); - data.forEach(msg -> sender.tell(new RpcSessionTellMsg(msg), ActorRef.noSender())); - } else { - log.info("[{}][{}] No pending messages to forward.", remoteAddress, uuid); - } - } - - private ActorRef createSessionActor(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Creating session actor.", msg.getMsgUid()); - ActorRef actor = context().actorOf( - Props.create(new RpcSessionActor.ActorCreator(systemContext, msg.getMsgUid())) - .withDispatcher(DefaultActorService.RPC_DISPATCHER_NAME)); - actor.tell(msg, context().self()); - return actor; - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - public ActorCreator(ActorSystemContext context) { - super(context); - } - - @Override - public RpcManagerActor create() { - return new RpcManagerActor(context); - } - } - - @Override - public SupervisorStrategy supervisorStrategy() { - return strategy; - } - - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { - log.warn("Unknown failure", t); - return SupervisorStrategy.resume(); - }); -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java deleted file mode 100644 index af2a7f0631..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; - -import java.util.UUID; - -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class RpcSessionActor extends ContextAwareActor { - - - private final UUID sessionId; - private GrpcSession session; - private GrpcSessionListener listener; - - private RpcSessionActor(ActorSystemContext systemContext, UUID sessionId) { - super(systemContext); - this.sessionId = sessionId; - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - tell((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - initSession((RpcSessionCreateRequestMsg) msg); - } - } - - private void tell(ClusterAPIProtos.ClusterMessage msg) { - if (session != null) { - session.sendMsg(msg); - } else { - log.trace("Failed to send message due to missing session!"); - } - } - - @Override - public void postStop() { - if (session != null) { - log.info("Closing session -> {}", session.getRemoteServer()); - try { - session.close(); - } catch (RuntimeException e) { - log.trace("Failed to close session!", e); - } - } - } - - private void initSession(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Initializing session", context().self()); - ServerAddress remoteServer = msg.getRemoteAddress(); - listener = new BasicRpcSessionListener(systemContext, context().parent(), context().self()); - if (msg.getRemoteAddress() == null) { - // Server session - session = new GrpcSession(listener); - session.setOutputStream(msg.getResponseObserver()); - session.initInputStream(); - session.initOutputStream(); - systemContext.getRpcService().onSessionCreated(msg.getMsgUid(), session.getInputStream()); - } else { - // Client session - ManagedChannel channel = ManagedChannelBuilder.forAddress(remoteServer.getHost(), remoteServer.getPort()).usePlaintext().build(); - session = new GrpcSession(remoteServer, listener, channel); - session.initInputStream(); - - ClusterRpcServiceGrpc.ClusterRpcServiceStub stub = ClusterRpcServiceGrpc.newStub(channel); - StreamObserver outputStream = stub.handleMsgs(session.getInputStream()); - - session.setOutputStream(outputStream); - session.initOutputStream(); - outputStream.onNext(toConnectMsg()); - } - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - private final UUID sessionId; - - public ActorCreator(ActorSystemContext context, UUID sessionId) { - super(context); - this.sessionId = sessionId; - } - - @Override - public RpcSessionActor create() { - return new RpcSessionActor(context, sessionId); - } - } - - private ClusterAPIProtos.ClusterMessage toConnectMsg() { - ServerAddress instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - return ClusterAPIProtos.ClusterMessage.newBuilder().setMessageType(CONNECT_RPC_MESSAGE).setServerAddress( - ClusterAPIProtos.ServerAddress.newBuilder().setHost(instance.getHost()) - .setPort(instance.getPort()).build()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index b700007498..6a3994a36f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -17,18 +17,13 @@ package org.thingsboard.server.actors.ruleChain; import akka.actor.ActorRef; import com.datastax.driver.core.ResultSetFuture; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.util.StringUtils; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -40,19 +35,15 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -67,11 +58,13 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; import java.util.Collections; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -79,6 +72,7 @@ import java.util.function.Consumer; /** * Created by ashvayka on 19.03.18. */ +@Slf4j class DefaultTbContext implements TbContext { public final static ObjectMapper mapper = new ObjectMapper(); @@ -91,6 +85,11 @@ class DefaultTbContext implements TbContext { this.nodeCtx = nodeCtx; } + @Override + public void tellSuccess(TbMsg msg) { + tellNext(msg, Collections.singleton(TbRelationTypes.SUCCESS), null); + } + @Override public void tellNext(TbMsg msg, String relationType) { tellNext(msg, Collections.singleton(relationType), null); @@ -101,16 +100,11 @@ class DefaultTbContext implements TbContext { tellNext(msg, relationTypes, null); } - @Override - public void tellNext(TbMsg msg, String relationType, Throwable th) { - tellNext(msg, Collections.singleton(relationType), th); - } - private void tellNext(TbMsg msg, Set relationTypes, Throwable th) { if (nodeCtx.getSelf().isDebugMode()) { relationTypes.forEach(relationType -> mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, relationType, th)); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); } @Override @@ -119,10 +113,94 @@ class DefaultTbContext implements TbContext { scheduleMsgWithDelay(new RuleNodeToSelfMsg(msg), delayMs, nodeCtx.getSelfActor()); } + @Override + public void enqueue(TbMsg tbMsg, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueue(tpi, tbMsg, onFailure, onSuccess); + } + + @Override + public void enqueue(TbMsg tbMsg, String queueName, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueue(tpi, tbMsg, onFailure, onSuccess); + } + + private void enqueue(TopicPartitionInfo tpi, TbMsg tbMsg, Consumer onFailure, Runnable onSuccess) { + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, new SimpleTbQueueCallback(onSuccess, onFailure)); + } + + @Override + public void enqueueForTellFailure(TbMsg tbMsg, String failureMessage) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(TbRelationTypes.FAILURE), failureMessage, null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String relationType) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String relationType, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); + } + + private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, String failureMessage, Runnable onSuccess, Consumer onFailure) { + RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId(); + RuleNodeId ruleNodeId = nodeCtx.getSelf().getId(); + tbMsg = TbMsg.newMsg(tbMsg, ruleChainId, ruleNodeId); + TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)) + .addAllRelationTypes(relationTypes); + if (failureMessage != null) { + msg.setFailureMessage(failureMessage); + } + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg.build(), new SimpleTbQueueCallback(onSuccess, onFailure)); + } + + @Override + public void ack(TbMsg tbMsg) { + if (nodeCtx.getSelf().isDebugMode()) { + mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "ACK", null); + } + tbMsg.getCallback().onSuccess(); + } + @Override public boolean isLocalEntity(EntityId entityId) { - Optional address = mainCtx.getRoutingService().resolveById(entityId); - return !address.isPresent(); + return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -134,66 +212,48 @@ class DefaultTbContext implements TbContext { if (nodeCtx.getSelf().isDebugMode()) { mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, TbRelationTypes.FAILURE, th); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), msg), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), + msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); } - @Override public void updateSelf(RuleNode self) { nodeCtx.setSelf(self); } @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), mainCtx.getQueuePartitionId()); + return TbMsg.newMsg(type, originator, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); } @Override public TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), mainCtx.getQueuePartitionId()); - } - - @Override - public void sendTbMsgToRuleEngine(TbMsg msg) { - mainCtx.getActorService().onMsg(new SendToClusterMsg(msg.getOriginator(), new ServiceToRuleEngineMsg(getTenantId(), msg))); + return TbMsg.transformMsg(origMsg, type, originator, metaData, data); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(customer); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, customer.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process customer created msg: " + e); - } + return entityCreatedMsg(customer, customer.getId(), ruleNodeId); } public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(device); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, device.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process device created msg: " + e); - } + return entityCreatedMsg(device, device.getId(), ruleNodeId); } public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(asset); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, asset.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process asset created msg: " + e); - } + return entityCreatedMsg(asset, asset.getId(), ruleNodeId); } public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) { + return entityCreatedMsg(alarm, alarm.getId(), ruleNodeId); + } + + public TbMsg entityCreatedMsg(E entity, I id, RuleNodeId ruleNodeId) { try { - ObjectNode entityNode = mapper.valueToTree(alarm); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, alarm.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + return TbMsg.newMsg(DataConstants.ENTITY_CREATED, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process alarm created msg: " + e); + throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " created msg: " + e); } } - @Override public RuleNodeId getSelfId() { return nodeCtx.getSelf().getId(); @@ -251,8 +311,8 @@ class DefaultTbContext implements TbContext { } @Override - public String getNodeId() { - return mainCtx.getNodeIdProvider().getNodeId(); + public String getServiceId() { + return mainCtx.getServiceInfoProvider().getServiceId(); } @Override @@ -320,11 +380,6 @@ class DefaultTbContext implements TbContext { return mainCtx.getEntityViewService(); } - @Override - public RuleChainTransactionService getRuleChainTransactionService() { - return mainCtx.getRuleChainTransactionService(); - } - @Override public EventLoopGroup getSharedEventLoop() { return mainCtx.getSharedEventLoopGroupService().getSharedEventLoopGroup(); @@ -341,35 +396,7 @@ class DefaultTbContext implements TbContext { @Override public RuleEngineRpcService getRpcService() { - return new RuleEngineRpcService() { - @Override - public void sendRpcReply(DeviceId deviceId, int requestId, String body) { - mainCtx.getDeviceRpcService().sendReplyToRpcCallFromDevice(nodeCtx.getTenantId(), deviceId, requestId, body); - } - - @Override - public void sendRpcRequest(RuleEngineDeviceRpcRequest src, Consumer consumer) { - ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), nodeCtx.getTenantId(), src.getDeviceId(), - src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); - mainCtx.getDeviceRpcService().forwardServerSideRPCRequestToDeviceActor(request, response -> { - if (src.isRestApiCall()) { - ServerAddress requestOriginAddress; - if (!StringUtils.isEmpty(src.getOriginHost())) { - requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); - } else { - requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); - } - mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); - } - consumer.accept(RuleEngineDeviceRpcResponse.builder() - .deviceId(src.getDeviceId()) - .requestId(src.getRequestId()) - .error(response.getError()) - .response(response.getResponse()) - .build()); - }); - } - }; + return mainCtx.getTbRuleEngineDeviceRpcService(); } @Override @@ -387,10 +414,6 @@ class DefaultTbContext implements TbContext { return mainCtx.getRedisTemplate(); } - @Override - public String getServerAddress() { - return mainCtx.getServerAddress(); - } private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) { TbMsgMetaData metaData = new TbMsgMetaData(); @@ -398,4 +421,29 @@ class DefaultTbContext implements TbContext { return metaData; } + private class SimpleTbQueueCallback implements TbQueueCallback { + private final Runnable onSuccess; + private final Consumer onFailure; + + public SimpleTbQueueCallback(Runnable onSuccess, Consumer onFailure) { + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (onSuccess != null) { + onSuccess.run(); + } + } + + @Override + public void onFailure(Throwable t) { + if (onFailure != null) { + onFailure.accept(t); + } else { + log.debug("[{}] Failed to put item into queue", nodeCtx.getTenantId(), t); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java deleted file mode 100644 index d062c5dc49..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.ruleChain; - -import lombok.Data; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; -import org.thingsboard.server.common.msg.aware.TenantAwareMsg; - -import java.io.Serializable; - -/** - * Created by ashvayka on 19.03.18. - */ -@Data -final class RemoteToRuleChainTellNextMsg extends RuleNodeToRuleChainTellNextMsg implements TenantAwareMsg, RuleChainAwareMsg { - - private static final long serialVersionUID = 2459605482321657447L; - private final TenantId tenantId; - private final RuleChainId ruleChainId; - - public RemoteToRuleChainTellNextMsg(RuleNodeToRuleChainTellNextMsg original, TenantId tenantId, RuleChainId ruleChainId) { - super(original.getOriginator(), original.getRelationTypes(), original.getMsg()); - this.tenantId = tenantId; - this.ruleChainId = ruleChainId; - } - - @Override - public MsgType getMsgType() { - return MsgType.REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 72faaef533..027f4aac9d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -15,25 +15,25 @@ */ package org.thingsboard.server.actors.ruleChain; -import akka.actor.ActorInitializationException; import akka.actor.OneForOneStrategy; import akka.actor.SupervisorStrategy; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.service.ComponentActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import scala.concurrent.duration.Duration; public class RuleChainActor extends ComponentActor { - private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId) { - super(systemContext, tenantId, ruleChainId); - setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChainId, systemContext, + private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChain ruleChain) { + super(systemContext, tenantId, ruleChain.getId()); + setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChain, systemContext, context().parent(), context().self())); } @@ -43,20 +43,17 @@ public class RuleChainActor extends ComponentActor { - private static final long DEFAULT_CLUSTER_PARTITION = 0L; private final ActorRef parent; private final ActorRef self; private final Map nodeActors; private final Map> nodeRoutes; private final RuleChainService service; + private final TbClusterService clusterService; + private String ruleChainName; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; - private String ruleChainName; - RuleChainActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, ActorSystemContext systemContext + RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { - super(systemContext, tenantId, ruleChainId); + super(systemContext, tenantId, ruleChain.getId()); + this.ruleChainName = ruleChain.getName(); this.parent = parent; this.self = self; this.nodeActors = new HashMap<>(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.ruleChainName = ruleChainId.toString(); + this.clusterService = systemContext.getClusterService(); } @Override @@ -92,7 +96,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor ruleNodeList = service.getRuleChainNodes(tenantId, entityId); log.trace("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); // Creating and starting the actors; @@ -152,15 +155,15 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actorRef.tell(msg, self)); } private ActorRef createRuleNodeActor(ActorContext context, RuleNode ruleNode) { String dispatcherName = tenantId.getId().equals(EntityId.NULL_UUID) ? DefaultActorService.SYSTEM_RULE_DISPATCHER_NAME : DefaultActorService.TENANT_RULE_DISPATCHER_NAME; return context.actorOf( - Props.create(new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleNode.getId())) + Props.create(new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleNode.getName(), ruleNode.getId())) .withDispatcher(dispatcherName), ruleNode.getId().toString()); } @@ -192,100 +195,131 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor address = systemContext.getRoutingService().resolveById(originatorEntityId); + onTellNext(envelope.getMsg(), envelope.getOriginator(), envelope.getRelationTypes(), envelope.getFailureMessage()); + } - if (address.isPresent()) { - onRemoteTellNext(address.get(), envelope); - } else { - onLocalTellNext(envelope); + private void onTellNext(TbMsg msg, RuleNodeId originatorNodeId, Set relationTypes, String failureMessage) { + try { + checkActive(msg); + EntityId entityId = msg.getOriginator(); + TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + List relations = nodeRoutes.get(originatorNodeId).stream() + .filter(r -> contains(relationTypes, r.getType())) + .collect(Collectors.toList()); + int relationsCount = relations.size(); + if (relationsCount == 0) { + log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); + if (relationTypes.contains(TbRelationTypes.FAILURE)) { + RuleNodeCtx ruleNodeCtx = nodeActors.get(originatorNodeId); + if (ruleNodeCtx != null) { + msg.getCallback().onFailure(new RuleNodeException(failureMessage, ruleChainName, ruleNodeCtx.getSelf())); + } else { + log.debug("[{}] Failure during message processing by Rule Node [{}]. Enable and see debug events for more info", entityId, originatorNodeId.getId()); + msg.getCallback().onFailure(new RuleEngineException("Failure during message processing by Rule Node [" + originatorNodeId.getId().toString() + "]")); + } + } else { + msg.getCallback().onSuccess(); + } + } else if (relationsCount == 1) { + for (RuleNodeRelation relation : relations) { + log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); + pushToTarget(tpi, msg, relation.getOut(), relation.getType()); + } + } else { + MultipleTbQueueTbMsgCallbackWrapper callbackWrapper = new MultipleTbQueueTbMsgCallbackWrapper(relationsCount, msg.getCallback()); + log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relations); + for (RuleNodeRelation relation : relations) { + EntityId target = relation.getOut(); + putToQueue(tpi, msg, callbackWrapper, target); + } + } + } catch (RuleNodeException rne) { + msg.getCallback().onFailure(rne); + } catch (Exception e) { + msg.getCallback().onFailure(new RuleEngineException("onTellNext - " + e.getMessage())); } } - private void onRemoteTellNext(ServerAddress serverAddress, RuleNodeToRuleChainTellNextMsg envelope) { - TbMsg msg = envelope.getMsg(); - log.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator()); - envelope = new RemoteToRuleChainTellNextMsg(envelope, tenantId, entityId); - systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); + private void putToQueue(TopicPartitionInfo tpi, TbMsg msg, TbQueueCallback callbackWrapper, EntityId target) { + switch (target.getEntityType()) { + case RULE_NODE: + putToQueue(tpi, msg.copyWithRuleNodeId(entityId, new RuleNodeId(target.getId())), callbackWrapper); + break; + case RULE_CHAIN: + putToQueue(tpi, msg.copyWithRuleChainId(new RuleChainId(target.getId())), callbackWrapper); + break; + } } - private void onLocalTellNext(RuleNodeToRuleChainTellNextMsg envelope) { - TbMsg msg = envelope.getMsg(); - RuleNodeId originatorNodeId = envelope.getOriginator(); - List relations = nodeRoutes.get(originatorNodeId).stream() - .filter(r -> contains(envelope.getRelationTypes(), r.getType())) - .collect(Collectors.toList()); - int relationsCount = relations.size(); - EntityId ackId = msg.getRuleNodeId() != null ? msg.getRuleNodeId() : msg.getRuleChainId(); - if (relationsCount == 0) { - log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); - if (ackId != null) { -// TODO: Ack this message in Kafka -// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); - } - } else if (relationsCount == 1) { - for (RuleNodeRelation relation : relations) { - log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - pushToTarget(msg, relation.getOut(), relation.getType()); + private void pushToTarget(TopicPartitionInfo tpi, TbMsg msg, EntityId target, String fromRelationType) { + if (tpi.isMyPartition()) { + switch (target.getEntityType()) { + case RULE_NODE: + pushMsgToNode(nodeActors.get(new RuleNodeId(target.getId())), msg, fromRelationType); + break; + case RULE_CHAIN: + parent.tell(new RuleChainToRuleChainMsg(new RuleChainId(target.getId()), entityId, msg, fromRelationType), self); + break; } } else { - for (RuleNodeRelation relation : relations) { - EntityId target = relation.getOut(); - log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - switch (target.getEntityType()) { - case RULE_NODE: - enqueueAndForwardMsgCopyToNode(msg, target, relation.getType()); - break; - case RULE_CHAIN: - enqueueAndForwardMsgCopyToChain(msg, target, relation.getType()); - break; - } - } - //TODO: Ideally this should happen in async way when all targets confirm that the copied messages are successfully written to corresponding target queues. - if (ackId != null) { -// TODO: Ack this message in Kafka -// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); - } + putToQueue(tpi, msg, new TbQueueTbMsgCallbackWrapper(msg.getCallback()), target); } } + private void putToQueue(TopicPartitionInfo tpi, TbMsg newMsg, TbQueueCallback callbackWrapper) { + ToRuleEngineMsg toQueueMsg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(newMsg)) + .build(); + clusterService.pushMsgToRuleEngine(tpi, newMsg.getId(), toQueueMsg, callbackWrapper); + } + private boolean contains(Set relationTypes, String type) { if (relationTypes == null) { return true; @@ -298,38 +332,18 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actors; + @Getter + protected RuleChain rootChain; + @Getter + protected ActorRef rootChainActor; - public RuleChainManagerActor(ActorSystemContext systemContext, RuleChainManager ruleChainManager) { + public RuleChainManagerActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); - this.ruleChainManager = ruleChainManager; + this.tenantId = tenantId; + this.actors = HashBiMap.create(); this.ruleChainService = systemContext.getRuleChainService(); } protected void initRuleChains() { - ruleChainManager.init(this.context()); + for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) { + RuleChainId ruleChainId = ruleChain.getId(); + log.debug("[{}|{}] Creating rule chain actor", ruleChainId.getEntityType(), ruleChain.getId()); + //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. + ActorRef actorRef = getOrCreateActor(this.context(), ruleChainId, id -> ruleChain); + visit(ruleChain, actorRef); + log.debug("[{}|{}] Rule Chain actor created.", ruleChainId.getEntityType(), ruleChainId.getId()); + } + } + + protected void visit(RuleChain entity, ActorRef actorRef) { + if (entity != null && entity.isRoot()) { + rootChain = entity; + rootChainActor = actorRef; + } + } + + public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId) { + return getOrCreateActor(context, ruleChainId, eId -> ruleChainService.findRuleChainById(TenantId.SYS_TENANT_ID, eId)); + } + + public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId, Function provider) { + return actors.computeIfAbsent(ruleChainId, eId -> { + RuleChain ruleChain = provider.apply(eId); + return context.actorOf(Props.create(new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain)) + .withDispatcher(DefaultActorService.TENANT_RULE_DISPATCHER_NAME), eId.toString()); + }); } protected ActorRef getEntityActorRef(EntityId entityId) { ActorRef target = null; - switch (entityId.getEntityType()) { - case RULE_CHAIN: - target = ruleChainManager.getOrCreateActor(this.context(), (RuleChainId) entityId); - break; + if (entityId.getEntityType() == EntityType.RULE_CHAIN) { + target = getOrCreateActor(this.context(), (RuleChainId) entityId); } return target; } protected void broadcast(Object msg) { - ruleChainManager.broadcast(msg); + actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } + + public ActorRef get(RuleChainId id) { + return actors.get(id); + } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java index e081768d93..0122471c52 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java @@ -32,7 +32,6 @@ public final class RuleChainToRuleChainMsg implements TbActorMsg, RuleChainAware private final RuleChainId source; private final TbMsg msg; private final String fromRelationType; - private final boolean enqueue; @Override public RuleChainId getRuleChainId() { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java index 65227818e8..74316a996b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java @@ -23,15 +23,18 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; public class RuleNodeActor extends ComponentActor { + private final String ruleChainName; private final RuleChainId ruleChainId; - private RuleNodeActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + private RuleNodeActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId, String ruleChainName, RuleNodeId ruleNodeId) { super(systemContext, tenantId, ruleNodeId); + this.ruleChainName = ruleChainName; this.ruleChainId = ruleChainId; - setProcessor(new RuleNodeActorMessageProcessor(tenantId, ruleChainId, ruleNodeId, systemContext, + setProcessor(new RuleNodeActorMessageProcessor(tenantId, this.ruleChainName, ruleNodeId, systemContext, context().parent(), context().self())); } @@ -53,6 +56,9 @@ public class RuleNodeActor extends ComponentActor { - private final ActorRef parent; + private final String ruleChainName; private final ActorRef self; - private final RuleChainService service; private RuleNode ruleNode; private TbNode tbNode; - private TbContext defaultCtx; + private DefaultTbContext defaultCtx; - RuleNodeActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, ActorSystemContext systemContext + RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { super(systemContext, tenantId, ruleNodeId); - this.parent = parent; + this.ruleChainName = ruleChainName; this.self = self; - this.service = systemContext.getRuleChainService(); this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId); this.defaultCtx = new DefaultTbContext(systemContext, new RuleNodeCtx(tenantId, parent, self, ruleNode)); } @@ -63,8 +59,8 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor relationTypes; private final TbMsg msg; + private final String failureMessage; @Override public MsgType getMsgType() { diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index 3032b1dfe0..f6403c5d56 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -15,24 +15,7 @@ */ package org.thingsboard.server.actors.service; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; -import org.thingsboard.server.common.transport.SessionMsgProcessor; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.rpc.RpcMsgListener; +public interface ActorService { -public interface ActorService extends SessionMsgProcessor, RpcMsgListener, DiscoveryServiceListener { - - void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); - - void onMsg(SendToClusterMsg msg); - - void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId); - - void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType); } diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java b/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java index 3260c7ee31..2a92caf4f6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java @@ -22,7 +22,7 @@ import org.thingsboard.server.actors.stats.StatsPersistMsg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; /** @@ -115,9 +115,9 @@ public abstract class ComponentActor status = system.terminate(); @@ -132,157 +97,4 @@ public class DefaultActorService implements ActorService { } } - @Override - public void onMsg(SendToClusterMsg msg) { - appActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onServerAdded(ServerInstance server) { - log.trace("Processing onServerAdded msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), true)); - } - - @Override - public void onServerUpdated(ServerInstance server) { - //Do nothing - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.trace("Processing onServerRemoved msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), false)); - } - - @Override - public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { - log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); - broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); - } - - @Override - public void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId) { - DeviceCredentialsUpdateNotificationMsg msg = new DeviceCredentialsUpdateNotificationMsg(tenantId, deviceId); - appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); - } - - @Override - public void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType) { - log.trace("[{}] Processing onDeviceNameOrTypeUpdate event, deviceName: {}, deviceType: {}", deviceId, deviceName, deviceType); - DeviceNameOrTypeUpdateMsg msg = new DeviceNameOrTypeUpdateMsg(tenantId, deviceId, deviceName, deviceType); - appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); - } - - public void broadcast(ToAllNodesMsg msg) { - actorContext.getEncodingService().encode(msg); - rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage - .newBuilder() - .setPayload(ByteString - .copyFrom(actorContext.getEncodingService().encode(msg))) - .setMessageType(CLUSTER_ACTOR_MESSAGE) - .build())); - appActor.tell(msg, ActorRef.noSender()); - } - - private void broadcast(ClusterEventMsg msg) { - this.appActor.tell(msg, ActorRef.noSender()); - this.rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Value("${cluster.stats.enabled:false}") - private boolean statsEnabled; - - private final AtomicInteger sentClusterMsgs = new AtomicInteger(0); - private final AtomicInteger receivedClusterMsgs = new AtomicInteger(0); - - - @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - int sent = sentClusterMsgs.getAndSet(0); - int received = receivedClusterMsgs.getAndSet(0); - if (sent > 0 || received > 0) { - log.info("Cluster msgs sent [{}] received [{}]", sent, received); - } - } - } - - @Override - public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - receivedClusterMsgs.incrementAndGet(); - } - ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); - if (log.isDebugEnabled()) { - log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); - log.info("MSG: {}", msg); - } - switch (msg.getMessageType()) { - case CLUSTER_ACTOR_MESSAGE: - java.util.Optional decodedMsg = actorContext.getEncodingService() - .decode(msg.getPayload().toByteArray()); - if (decodedMsg.isPresent()) { - appActor.tell(decodedMsg.get(), ActorRef.noSender()); - } else { - log.error("Error during decoding cluster proto message"); - } - break; - case TO_ALL_NODES_MSG: - //TODO - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: - actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: - actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: - actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TRANSACTION_SERVICE_MESSAGE: - actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); - break; - } - } - - @Override - public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onBroadcastMsg(RpcBroadcastMsg msg) { - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onDeviceAdded(Device device) { - deviceStateService.onDeviceAdded(device); - } } diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java index 2fdc918f8e..45779a9f89 100644 --- a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java @@ -16,9 +16,6 @@ package org.thingsboard.server.actors.shared; import akka.actor.ActorContext; -import akka.event.LoggingAdapter; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.stats.StatsPersistTick; @@ -26,10 +23,9 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; - -import javax.annotation.Nullable; -import java.util.function.Consumer; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.RuleNodeException; @Slf4j public abstract class ComponentMsgProcessor extends AbstractContextAwareMsgProcessor { @@ -50,7 +46,7 @@ public abstract class ComponentMsgProcessor extends Abstract public abstract void stop(ActorContext context) throws Exception; - public abstract void onClusterEventMsg(ClusterEventMsg msg) throws Exception; + public abstract void onPartitionChangeMsg(PartitionChangeMsg msg) throws Exception; public void onCreated(ActorContext context) throws Exception { start(context); @@ -81,11 +77,17 @@ public abstract class ComponentMsgProcessor extends Abstract schedulePeriodicMsgWithDelay(context, new StatsPersistTick(), statsPersistFrequency, statsPersistFrequency); } - protected void checkActive() { + protected void checkActive(TbMsg tbMsg) throws RuleNodeException { if (state != ComponentLifecycleState.ACTIVE) { log.debug("Component is not active. Current state [{}] for processor [{}][{}] tenant [{}]", state, entityId.getEntityType(), entityId, tenantId); - throw new IllegalStateException("Rule chain is not active! " + entityId + " - " + tenantId); + RuleNodeException ruleNodeException = getInactiveException(); + if (tbMsg != null) { + tbMsg.getCallback().onFailure(ruleNodeException); + } + throw ruleNodeException; } } + abstract protected RuleNodeException getInactiveException(); + } diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java deleted file mode 100644 index b824cf61ff..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared; - -import akka.actor.ActorContext; -import akka.actor.ActorRef; -import akka.actor.Props; -import akka.actor.UntypedActor; -import akka.japi.Creator; -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.common.data.SearchTextBased; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; -import org.thingsboard.server.common.data.page.PageDataIterable; - -import java.util.HashMap; -import java.util.Map; - -/** - * Created by ashvayka on 15.03.18. - */ -@Slf4j -public abstract class EntityActorsManager> { - - protected final ActorSystemContext systemContext; - protected final BiMap actors; - - public EntityActorsManager(ActorSystemContext systemContext) { - this.systemContext = systemContext; - this.actors = HashBiMap.create(); - } - - protected abstract TenantId getTenantId(); - - protected abstract String getDispatcherName(); - - protected abstract Creator creator(T entityId); - - protected abstract PageDataIterable.FetchFunction getFetchEntitiesFunction(); - - public void init(ActorContext context) { - for (M entity : new PageDataIterable<>(getFetchEntitiesFunction(), ContextAwareActor.ENTITY_PACK_LIMIT)) { - T entityId = (T) entity.getId(); - log.debug("[{}|{}] Creating entity actor", entityId.getEntityType(), entityId.getId()); - //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. - ActorRef actorRef = getOrCreateActor(context, entityId); - visit(entity, actorRef); - log.debug("[{}|{}] Entity actor created.", entityId.getEntityType(), entityId.getId()); - } - } - - public void visit(M entity, ActorRef actorRef) { - } - - public ActorRef getOrCreateActor(ActorContext context, T entityId) { - return actors.computeIfAbsent(entityId, eId -> - context.actorOf(Props.create(creator(eId)) - .withDispatcher(getDispatcherName()), eId.toString())); - } - - public void broadcast(Object msg) { - actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); - } - - public void remove(T id) { - actors.remove(id); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java deleted file mode 100644 index ef2b4282d4..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import akka.actor.ActorRef; -import akka.japi.Creator; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainActor; -import org.thingsboard.server.actors.shared.EntityActorsManager; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.dao.rule.RuleChainService; - -/** - * Created by ashvayka on 15.03.18. - */ -@Slf4j -public abstract class RuleChainManager extends EntityActorsManager { - - protected final RuleChainService service; - @Getter - protected RuleChain rootChain; - @Getter - protected ActorRef rootChainActor; - - public RuleChainManager(ActorSystemContext systemContext) { - super(systemContext); - this.service = systemContext.getRuleChainService(); - } - - @Override - public Creator creator(RuleChainId entityId) { - return new RuleChainActor.ActorCreator(systemContext, getTenantId(), entityId); - } - - @Override - public void visit(RuleChain entity, ActorRef actorRef) { - if (entity != null && entity.isRoot()) { - rootChain = entity; - rootChainActor = actorRef; - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java deleted file mode 100644 index f2a80a9e82..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; -import org.thingsboard.server.common.data.page.TextPageData; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.dao.model.ModelConstants; - -import java.util.Collections; - -public class SystemRuleChainManager extends RuleChainManager { - - public SystemRuleChainManager(ActorSystemContext systemContext) { - super(systemContext); - } - - @Override - protected FetchFunction getFetchEntitiesFunction() { - return link -> new TextPageData<>(Collections.emptyList(), link); - } - - @Override - protected TenantId getTenantId() { - return ModelConstants.SYSTEM_TENANT; - } - - @Override - protected String getDispatcherName() { - return DefaultActorService.SYSTEM_RULE_DISPATCHER_NAME; - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java deleted file mode 100644 index cfe862af49..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import akka.actor.ActorContext; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; -import org.thingsboard.server.common.data.rule.RuleChain; - -public class TenantRuleChainManager extends RuleChainManager { - - private final TenantId tenantId; - - public TenantRuleChainManager(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext); - this.tenantId = tenantId; - } - - @Override - public void init(ActorContext context) { - super.init(context); - } - - @Override - protected TenantId getTenantId() { - return tenantId; - } - - @Override - protected String getDispatcherName() { - return DefaultActorService.TENANT_RULE_DISPATCHER_NAME; - } - - @Override - protected FetchFunction getFetchEntitiesFunction() { - return link -> service.findTenantRuleChains(tenantId, link); - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 31461a8fac..717dcecc19 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -24,7 +24,6 @@ import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; @Slf4j public class StatsActor extends ContextAwareActor { @@ -58,12 +57,12 @@ public class StatsActor extends ContextAwareActor { event.setEntityId(msg.getEntityId()); event.setTenantId(msg.getTenantId()); event.setType(DataConstants.STATS); - event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); + event.setBody(toBodyJson(systemContext.getServiceInfoProvider().getServiceId(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); systemContext.getEventService().save(event); } - private JsonNode toBodyJson(ServerAddress server, long messagesProcessed, long errorsOccurred) { - return mapper.createObjectNode().put("server", server.toString()).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); + private JsonNode toBodyJson(String serviceId, long messagesProcessed, long errorsOccurred) { + return mapper.createObjectNode().put("server", serviceId).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); } public static class ActorCreator extends ContextBasedCreator { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 4aacfb55bf..3cee82127c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -22,41 +22,43 @@ import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; import akka.actor.Terminated; -import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.device.DeviceActorCreator; -import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.actors.shared.rulechain.TenantRuleChainManager; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceType; import scala.concurrent.duration.Duration; -import java.util.HashMap; -import java.util.Map; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; public class TenantActor extends RuleChainManagerActor { - private final TenantId tenantId; private final BiMap deviceActors; + private boolean isRuleEngineForCurrentTenant; + private boolean isCore; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext, new TenantRuleChainManager(systemContext, tenantId)); - this.tenantId = tenantId; + super(systemContext, tenantId); this.deviceActors = HashBiMap.create(); } @@ -65,12 +67,37 @@ public class TenantActor extends RuleChainManagerActor { return strategy; } + boolean cantFindTenant = false; + @Override public void preStart() { log.info("[{}] Starting tenant actor.", tenantId); try { - initRuleChains(); - log.info("[{}] Tenant actor started.", tenantId); + Tenant tenant = systemContext.getTenantService().findTenantById(tenantId); + if (tenant == null) { + cantFindTenant = true; + log.info("[{}] Started tenant actor for missing tenant.", tenantId); + } else { + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + + if (isRuleEngineForCurrentTenant) { + try { + if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { + log.info("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + isRuleEngineForCurrentTenant = false; + } + } catch (Exception e) { + cantFindTenant = true; + } + } + log.info("[{}] Tenant actor started.", tenantId); + } } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); } @@ -83,18 +110,36 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { + if (cantFindTenant) { + log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg); + if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { + QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; + queueMsg.getTbMsg().getCallback().onSuccess(); + } + return true; + } switch (msg.getMsgType()) { - case CLUSTER_EVENT_MSG: - broadcast(msg); + case PARTITION_CHANGE_MSG: + PartitionChangeMsg partitionChangeMsg = (PartitionChangeMsg) msg; + ServiceType serviceType = partitionChangeMsg.getServiceQueueKey().getServiceType(); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + //To Rule Chain Actors + broadcast(msg); + } else if (ServiceType.TB_CORE.equals(serviceType)) { + //To Device Actors + List repartitionedDevices = + deviceActors.keySet().stream().filter(deviceId -> !isMyPartition(deviceId)).collect(Collectors.toList()); + for (DeviceId deviceId : repartitionedDevices) { + ActorRef deviceActor = deviceActors.remove(deviceId); + context().stop(deviceActor); + } + } break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case SERVICE_TO_RULE_ENGINE_MSG: - onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); - break; - case DEVICE_ACTOR_TO_RULE_ENGINE_MSG: - onDeviceActorToRuleEngineMsg((DeviceActorToRuleEngineMsg) msg); + case QUEUE_TO_RULE_ENGINE_MSG: + onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -105,7 +150,6 @@ public class TenantActor extends RuleChainManagerActor { onToDeviceActorMsg((DeviceAwareMsg) msg); break; case RULE_CHAIN_TO_RULE_CHAIN_MSG: - case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; default: @@ -114,41 +158,59 @@ public class TenantActor extends RuleChainManagerActor { return true; } - private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { - if (ruleChainManager.getRootChainActor() != null) { - ruleChainManager.getRootChainActor().tell(msg, self()); - } else { - log.info("[{}] No Root Chain: {}", tenantId, msg); - } + private boolean isMyPartition(DeviceId deviceId) { + return systemContext.resolve(ServiceType.TB_CORE, tenantId, deviceId).isMyPartition(); } - private void onDeviceActorToRuleEngineMsg(DeviceActorToRuleEngineMsg msg) { - if (ruleChainManager.getRootChainActor() != null) { - ruleChainManager.getRootChainActor().tell(msg, self()); + private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { + if (!isRuleEngineForCurrentTenant) { + log.warn("RECEIVED INVALID MESSAGE: {}", msg); + return; + } + TbMsg tbMsg = msg.getTbMsg(); + if (tbMsg.getRuleChainId() == null) { + if (getRootChainActor() != null) { + getRootChainActor().tell(msg, self()); + } else { + tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); + log.info("[{}] No Root Chain: {}", tenantId, msg); + } } else { - log.info("[{}] No Root Chain: {}", tenantId, msg); + ActorRef ruleChainActor = get(tbMsg.getRuleChainId()); + if (ruleChainActor != null) { + ruleChainActor.tell(msg, self()); + } else { + log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId()); + //TODO: 3.1 Log it to dead letters queue; + tbMsg.getCallback().onSuccess(); + } } } private void onRuleChainMsg(RuleChainAwareMsg msg) { - ruleChainManager.getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); + getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); } private void onToDeviceActorMsg(DeviceAwareMsg msg) { + if (!isCore) { + log.warn("RECEIVED INVALID MESSAGE: {}", msg); + } getOrCreateDeviceActor(msg.getDeviceId()).tell(msg, ActorRef.noSender()); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { - ActorRef target = getEntityActorRef(msg.getEntityId()); - if (target != null) { - if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { - RuleChain ruleChain = systemContext.getRuleChainService(). - findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); - ruleChainManager.visit(ruleChain, target); + if (isRuleEngineForCurrentTenant) { + ActorRef target = getEntityActorRef(msg.getEntityId()); + if (target != null) { + if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { + RuleChain ruleChain = systemContext.getRuleChainService(). + findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); + visit(ruleChain, target); + } + target.tell(msg, ActorRef.noSender()); + } else { + log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } - target.tell(msg, ActorRef.noSender()); - } else { - log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } } @@ -172,7 +234,7 @@ public class TenantActor extends RuleChainManagerActor { if (removed) { log.debug("[{}] Removed actor:", terminated); } else { - log.warn("[{}] Removed actor was not found in the device map!"); + log.debug("Removed actor was not found in the device map!"); } } else { throw new IllegalStateException("Remote actors are not supported!"); @@ -195,15 +257,12 @@ public class TenantActor extends RuleChainManagerActor { } } - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), new Function() { - @Override - public SupervisorStrategy.Directive apply(Throwable t) { - log.warn("[{}] Unknown failure", tenantId, t); - if (t instanceof ActorInitializationException) { - return SupervisorStrategy.stop(); - } else { - return SupervisorStrategy.resume(); - } + private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { + log.warn("[{}] Unknown failure", tenantId, t); + if (t instanceof ActorInitializationException) { + return SupervisorStrategy.stop(); + } else { + return SupervisorStrategy.resume(); } }); diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 526f0b73c8..683345fe7f 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -39,6 +39,7 @@ import org.springframework.security.web.util.matcher.AntPathRequestMatcher; import org.springframework.web.cors.UrlBasedCorsConfigurationSource; import org.springframework.web.filter.CorsFilter; import org.thingsboard.server.dao.audit.AuditLogLevelFilter; +import org.thingsboard.server.dao.oauth2.OAuth2Configuration; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; import org.thingsboard.server.service.security.auth.jwt.JwtTokenAuthenticationProcessingFilter; @@ -73,12 +74,29 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt public static final String WS_TOKEN_BASED_AUTH_ENTRY_POINT = "/api/ws/**"; @Autowired private ThingsboardErrorResponseHandler restAccessDeniedHandler; - @Autowired private AuthenticationSuccessHandler successHandler; - @Autowired private AuthenticationFailureHandler failureHandler; + + @Autowired(required = false) + @Qualifier("oauth2AuthenticationSuccessHandler") + private AuthenticationSuccessHandler oauth2AuthenticationSuccessHandler; + + @Autowired(required = false) + @Qualifier("oauth2AuthenticationFailureHandler") + private AuthenticationFailureHandler oauth2AuthenticationFailureHandler; + + @Autowired + @Qualifier("defaultAuthenticationSuccessHandler") + private AuthenticationSuccessHandler successHandler; + + @Autowired + @Qualifier("defaultAuthenticationFailureHandler") + private AuthenticationFailureHandler failureHandler; + @Autowired private RestAuthenticationProvider restAuthenticationProvider; @Autowired private JwtAuthenticationProvider jwtAuthenticationProvider; @Autowired private RefreshTokenAuthenticationProvider refreshTokenAuthenticationProvider; + @Autowired(required = false) OAuth2Configuration oauth2Configuration; + @Autowired @Qualifier("jwtHeaderTokenExtractor") private TokenExtractor jwtHeaderTokenExtractor; @@ -107,7 +125,6 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt return filter; } - @Bean protected JwtTokenAuthenticationProcessingFilter buildJwtTokenAuthenticationProcessingFilter() throws Exception { List pathsToSkip = new ArrayList(Arrays.asList(NON_TOKEN_BASED_AUTH_ENTRY_POINTS)); pathsToSkip.addAll(Arrays.asList(WS_TOKEN_BASED_AUTH_ENTRY_POINT, TOKEN_REFRESH_ENTRY_POINT, FORM_BASED_LOGIN_ENTRY_POINT, @@ -190,9 +207,15 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt .addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildWsJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class); + if (oauth2Configuration != null && oauth2Configuration.isEnabled()) { + http.oauth2Login() + .loginPage("/oauth2Login") + .loginProcessingUrl(oauth2Configuration.getLoginProcessingUrl()) + .successHandler(oauth2AuthenticationSuccessHandler) + .failureHandler(oauth2AuthenticationFailureHandler); + } } - @Bean @ConditionalOnMissingBean(CorsFilter.class) public CorsFilter corsFilter(@Autowired MvcCorsProperties mvcCorsProperties) { diff --git a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java index 5e5bd94afa..e57695da69 100644 --- a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java @@ -32,11 +32,13 @@ import org.springframework.web.socket.server.support.HttpSessionHandshakeInterce import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.controller.plugin.TbWebSocketHandler; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; @Configuration +@TbCoreComponent @EnableWebSocket public class WebSocketConfiguration implements WebSocketConfigurer { @@ -62,7 +64,8 @@ public class WebSocketConfiguration implements WebSocketConfigurer { SecurityUser user = null; try { user = getCurrentUser(); - } catch (ThingsboardException ex) {} + } catch (ThingsboardException ex) { + } if (user == null) { response.setStatusCode(HttpStatus.UNAUTHORIZED); return false; diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index b7efc180a5..8873d82366 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -25,23 +25,25 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.common.data.security.model.SecuritySettings; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.update.UpdateService; -import org.thingsboard.server.common.data.UpdateMessage; @RestController +@TbCoreComponent @RequestMapping("/api/admin") public class AdminController extends BaseController { @Autowired private MailService mailService; - + @Autowired private AdminSettingsService adminSettingsService; @@ -65,7 +67,7 @@ public class AdminController extends BaseController { @PreAuthorize("hasAuthority('SYS_ADMIN')") @RequestMapping(value = "/settings", method = RequestMethod.POST) - @ResponseBody + @ResponseBody public AdminSettings saveAdminSettings(@RequestBody AdminSettings adminSettings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); @@ -111,8 +113,8 @@ public class AdminController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); adminSettings = checkNotNull(adminSettings); if (adminSettings.getKey().equals("mail")) { - String email = getCurrentUser().getEmail(); - mailService.sendTestMail(adminSettings.getJsonValue(), email); + String email = getCurrentUser().getEmail(); + mailService.sendTestMail(adminSettings.getJsonValue(), email); } } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 09e6dd5d10..2132675f97 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; @@ -41,10 +41,12 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController +@TbCoreComponent @RequestMapping("/api") public class AlarmController extends BaseController { @@ -118,7 +120,9 @@ public class AlarmController extends BaseController { try { AlarmId alarmId = new AlarmId(toUUID(strAlarmId)); Alarm alarm = checkAlarmId(alarmId, Operation.WRITE); - alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, System.currentTimeMillis()).get(); + long ackTs = System.currentTimeMillis(); + alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, ackTs).get(); + alarm.setAckTs(ackTs); logEntityAction(alarmId, alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_ACK, null); } catch (Exception e) { throw handleException(e); @@ -133,7 +137,9 @@ public class AlarmController extends BaseController { try { AlarmId alarmId = new AlarmId(toUUID(strAlarmId)); Alarm alarm = checkAlarmId(alarmId, Operation.WRITE); - alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, System.currentTimeMillis()).get(); + long clearTs = System.currentTimeMillis(); + alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, clearTs).get(); + alarm.setClearTs(clearTs); logEntityAction(alarmId, alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_CLEAR, null); } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 0fbec9560f..03c5462446 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -32,16 +32,15 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; -import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -51,6 +50,7 @@ import java.util.List; import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class AssetController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java index 56f730f0fb..c99ba8a018 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.Arrays; import java.util.List; @@ -39,6 +40,7 @@ import java.util.UUID; import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class AuditLogController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 0cb9b29501..9f6c6f3b44 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -38,8 +38,11 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.dao.oauth2.OAuth2Service; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; import org.thingsboard.server.common.data.security.model.SecuritySettings; @@ -49,13 +52,16 @@ import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.model.token.JwtToken; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.system.SystemSecurityService; +import org.thingsboard.server.utils.MiscUtils; import ua_parser.Client; import javax.servlet.http.HttpServletRequest; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; @RestController +@TbCoreComponent @RequestMapping("/api") @Slf4j public class AuthController extends BaseController { @@ -78,6 +84,9 @@ public class AuthController extends BaseController { @Autowired private AuditLogService auditLogService; + @Autowired + private OAuth2Service oauth2Service; + @PreAuthorize("isAuthenticated()") @RequestMapping(value = "/auth/user", method = RequestMethod.GET) public @ResponseBody User getUser() throws ThingsboardException { @@ -162,7 +171,7 @@ public class AuthController extends BaseController { try { String email = resetPasswordByEmailRequest.get("email").asText(); UserCredentials userCredentials = userService.requestPasswordReset(TenantId.SYS_TENANT_ID, email); - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String resetUrl = String.format("%s/api/noauth/resetPassword?resetToken=%s", baseUrl, userCredentials.getResetToken()); @@ -199,6 +208,7 @@ public class AuthController extends BaseController { @ResponseBody public JsonNode activateUser( @RequestBody JsonNode activateRequest, + @RequestParam(required = false, defaultValue = "true") boolean sendActivationMail, HttpServletRequest request) throws ThingsboardException { try { String activateToken = activateRequest.get("activateToken").asText(); @@ -209,14 +219,16 @@ public class AuthController extends BaseController { User user = userService.findUserById(TenantId.SYS_TENANT_ID, credentials.getUserId()); UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail()); SecurityUser securityUser = new SecurityUser(user, credentials.isEnabled(), principal); - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String loginUrl = String.format("%s/login", baseUrl); String email = user.getEmail(); - try { - mailService.sendAccountActivatedEmail(loginUrl, email); - } catch (Exception e) { - log.info("Unable to send account activation email [{}]", e.getMessage()); + if (sendActivationMail) { + try { + mailService.sendAccountActivatedEmail(loginUrl, email); + } catch (Exception e) { + log.info("Unable to send account activation email [{}]", e.getMessage()); + } } JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser); @@ -254,7 +266,7 @@ public class AuthController extends BaseController { User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId()); UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail()); SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), principal); - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String loginUrl = String.format("%s/login", baseUrl); String email = user.getEmail(); mailService.sendPasswordWasResetEmail(loginUrl, email); @@ -325,4 +337,13 @@ public class AuthController extends BaseController { } } + @RequestMapping(value = "/noauth/oauth2Clients", method = RequestMethod.POST) + @ResponseBody + public List getOAuth2Clients() throws ThingsboardException { + try { + return oauth2Service.getOAuth2Clients(); + } catch (Exception e) { + throw handleException(e); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 124a6e7ba1..a8d6098c4a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -27,7 +26,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.DashboardInfo; @@ -36,10 +34,11 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; @@ -71,8 +70,6 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -93,7 +90,11 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.component.ComponentDiscoveryService; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; @@ -112,6 +113,7 @@ import java.util.UUID; import static org.thingsboard.server.dao.service.Validator.validateId; @Slf4j +@TbCoreComponent public abstract class BaseController { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; @@ -162,7 +164,7 @@ public abstract class BaseController { protected RuleChainService ruleChainService; @Autowired - protected ActorService actorService; + protected TbClusterService tbClusterService; @Autowired protected RelationService relationService; @@ -185,6 +187,12 @@ public abstract class BaseController { @Autowired protected ClaimDevicesService claimDevicesService; + @Autowired + protected PartitionService partitionService; + + @Autowired + protected TbQueueProducerProvider producerProvider; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -513,39 +521,6 @@ public abstract class BaseController { return ruleNode; } - - protected String constructBaseUrl(HttpServletRequest request) { - String scheme = request.getScheme(); - - String forwardedProto = request.getHeader("x-forwarded-proto"); - if (forwardedProto != null) { - scheme = forwardedProto; - } - - int serverPort = request.getServerPort(); - if (request.getHeader("x-forwarded-port") != null) { - try { - serverPort = request.getIntHeader("x-forwarded-port"); - } catch (NumberFormatException e) { - } - } else if (forwardedProto != null) { - switch (forwardedProto) { - case "http": - serverPort = 80; - break; - case "https": - serverPort = 443; - break; - } - } - - String baseUrl = String.format("%s://%s:%d", - scheme, - request.getServerName(), - serverPort); - return baseUrl; - } - protected I emptyId(EntityType entityType) { return (I) EntityIdFactory.getByTypeAndUuid(entityType, ModelConstants.NULL_UUID); } @@ -642,6 +617,8 @@ public abstract class BaseController { entityNode.put(attr.getKey(), attr.getDoubleValue().get()); } else if (attr.getDataType() == DataType.LONG) { entityNode.put(attr.getKey(), attr.getLongValue().get()); + } else if (attr.getDataType() == DataType.JSON) { + entityNode.set(attr.getKey(), json.readTree(attr.getJsonValue().get())); } else { entityNode.put(attr.getKey(), attr.getValueAsString()); } @@ -657,10 +634,14 @@ public abstract class BaseController { } } } - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, entityId, metaData, TbMsgDataType.JSON - , json.writeValueAsString(entityNode) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(entityId, new ServiceToRuleEngineMsg(user.getTenantId(), tbMsg))); + TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); + TenantId tenantId = user.getTenantId(); + if (tenantId.isNullUid()) { + if (entity instanceof HasTenantId) { + tenantId = ((HasTenantId) entity).getTenantId(); + } + } + tbClusterService.pushMsgToRuleEngine(tenantId, entityId, tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java index ef9508ad7c..d75da8b34c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java +++ b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java @@ -25,12 +25,14 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.HashSet; import java.util.List; import java.util.Set; @RestController +@TbCoreComponent @RequestMapping("/api") public class ComponentDescriptorController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index def876e272..18fcfc2ab5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -36,10 +36,12 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController +@TbCoreComponent @RequestMapping("/api") public class CustomerController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index 741a9fec95..68f18c9081 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; @@ -41,6 +40,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -48,6 +48,7 @@ import java.util.HashSet; import java.util.Set; @RestController +@TbCoreComponent @RequestMapping("/api") public class DashboardController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 828f75bd47..ca61ecc8e9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -18,6 +18,7 @@ package org.thingsboard.server.controller; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; @@ -30,6 +31,9 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; +import org.thingsboard.server.common.data.ClaimRequest; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -44,11 +48,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.common.data.ClaimRequest; import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -60,6 +64,7 @@ import java.util.List; import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class DeviceController extends BaseController { @@ -94,12 +99,8 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - actorService - .onDeviceNameOrTypeUpdate( - savedDevice.getTenantId(), - savedDevice.getId(), - savedDevice.getName(), - savedDevice.getType()); + tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), + savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); logEntityAction(savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), @@ -252,7 +253,9 @@ public class DeviceController extends BaseController { try { Device device = checkDeviceId(deviceCredentials.getDeviceId(), Operation.WRITE_CREDENTIALS); DeviceCredentials result = checkNotNull(deviceCredentialsService.updateDeviceCredentials(getCurrentUser().getTenantId(), deviceCredentials)); - actorService.onCredentialsUpdate(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()); + + tbClusterService.pushMsgToCore(new DeviceCredentialsUpdateNotificationMsg(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()), null); + logEntityAction(device.getId(), device, device.getCustomerId(), ActionType.CREDENTIALS_UPDATED, null, deviceCredentials); @@ -390,7 +393,7 @@ public class DeviceController extends BaseController { } } - @PreAuthorize("hasAnyAuthority('CUSTOMER_USER')") + @PreAuthorize("hasAuthority('CUSTOMER_USER')") @RequestMapping(value = "/customer/device/{deviceName}/claim", method = RequestMethod.POST) @ResponseBody public DeferredResult claimDevice(@PathVariable(DEVICE_NAME) String deviceName, @@ -425,11 +428,12 @@ public class DeviceController extends BaseController { deferredResult.setResult(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); } } + @Override public void onFailure(Throwable t) { deferredResult.setErrorResult(t); } - }); + }, MoreExecutors.directExecutor()); return deferredResult; } catch (Exception e) { throw handleException(e); @@ -466,7 +470,7 @@ public class DeviceController extends BaseController { public void onFailure(Throwable t) { deferredResult.setErrorResult(t); } - }); + }, MoreExecutors.directExecutor()); return deferredResult; } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java index ea87f86781..88aa9ce850 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java @@ -24,23 +24,24 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import java.util.List; +import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class EntityRelationController extends BaseController { @@ -167,7 +168,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findByFrom(getTenantId(), entityId, typeGroup)); + return checkNotNull(filterRelationsByReadPermission(relationService.findByFrom(getTenantId(), entityId, typeGroup))); } catch (Exception e) { throw handleException(e); } @@ -185,7 +186,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findInfoByFrom(getTenantId(), entityId, typeGroup).get()); + return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByFrom(getTenantId(), entityId, typeGroup).get())); } catch (Exception e) { throw handleException(e); } @@ -205,7 +206,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findByFromAndType(getTenantId(), entityId, strRelationType, typeGroup)); + return checkNotNull(filterRelationsByReadPermission(relationService.findByFromAndType(getTenantId(), entityId, strRelationType, typeGroup))); } catch (Exception e) { throw handleException(e); } @@ -223,7 +224,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findByTo(getTenantId(), entityId, typeGroup)); + return checkNotNull(filterRelationsByReadPermission(relationService.findByTo(getTenantId(), entityId, typeGroup))); } catch (Exception e) { throw handleException(e); } @@ -241,7 +242,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findInfoByTo(getTenantId(), entityId, typeGroup).get()); + return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByTo(getTenantId(), entityId, typeGroup).get())); } catch (Exception e) { throw handleException(e); } @@ -261,7 +262,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findByToAndType(getTenantId(), entityId, strRelationType, typeGroup)); + return checkNotNull(filterRelationsByReadPermission(relationService.findByToAndType(getTenantId(), entityId, strRelationType, typeGroup))); } catch (Exception e) { throw handleException(e); } @@ -276,7 +277,7 @@ public class EntityRelationController extends BaseController { checkNotNull(query.getFilters()); checkEntityId(query.getParameters().getEntityId(), Operation.READ); try { - return checkNotNull(relationService.findByQuery(getTenantId(), query).get()); + return checkNotNull(filterRelationsByReadPermission(relationService.findByQuery(getTenantId(), query).get())); } catch (Exception e) { throw handleException(e); } @@ -291,12 +292,28 @@ public class EntityRelationController extends BaseController { checkNotNull(query.getFilters()); checkEntityId(query.getParameters().getEntityId(), Operation.READ); try { - return checkNotNull(relationService.findInfoByQuery(getTenantId(), query).get()); + return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByQuery(getTenantId(), query).get())); } catch (Exception e) { throw handleException(e); } } + private List filterRelationsByReadPermission(List relationsByQuery) { + return relationsByQuery.stream().filter(relationByQuery -> { + try { + checkEntityId(relationByQuery.getTo(), Operation.READ); + } catch (ThingsboardException e) { + return false; + } + try { + checkEntityId(relationByQuery.getFrom(), Operation.READ); + } catch (ThingsboardException e) { + return false; + } + return true; + }).collect(Collectors.toList()); + } + private RelationTypeGroup parseRelationTypeGroup(String strRelationTypeGroup, RelationTypeGroup defaultValue) { RelationTypeGroup result = defaultValue; if (strRelationTypeGroup != null && strRelationTypeGroup.trim().length() > 0) { diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java index 2f146e5738..05c57d35fa 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java @@ -18,6 +18,7 @@ package org.thingsboard.server.controller; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; @@ -47,6 +48,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -64,6 +66,7 @@ import static org.thingsboard.server.controller.CustomerController.CUSTOMER_ID; * Created by Victor Basanets on 8/28/2017. */ @RestController +@TbCoreComponent @RequestMapping("/api") @Slf4j public class EntityViewController extends BaseController { @@ -158,7 +161,7 @@ public class EntityViewController extends BaseController { }); } return null; - }); + }, MoreExecutors.directExecutor()); } else { return Futures.immediateFuture(null); } diff --git a/application/src/main/java/org/thingsboard/server/controller/EventController.java b/application/src/main/java/org/thingsboard/server/controller/EventController.java index d9ad9d54bf..500549c09a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EventController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EventController.java @@ -31,9 +31,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; @RestController +@TbCoreComponent @RequestMapping("/api") public class EventController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java new file mode 100644 index 0000000000..a0b46f9166 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/QueueController.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +public class QueueController extends BaseController { + + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/tenant/queues", params = {"serviceType"}, method = RequestMethod.GET) + @ResponseBody + public List getTenantQueuesByServiceType(@RequestParam String serviceType) throws ThingsboardException { + checkParameter("serviceType", serviceType); + try { + ServiceType type = ServiceType.valueOf(serviceType); + switch (type) { + case TB_RULE_ENGINE: + return Arrays.asList("Main", "HighPriority", "SequentialByOriginator"); + default: + return Collections.emptyList(); + } + } catch (Exception e) { + throw handleException(e); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index a8298ce82b..b88c3f3695 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -31,7 +31,6 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; @@ -43,27 +42,25 @@ import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.rpc.RpcRequest; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.service.rpc.DeviceRpcService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.LocalRequestMetaData; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.telemetry.exception.ToErrorResponseEntity; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.io.IOException; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; /** * Created by ashvayka on 22.03.18. */ @RestController +@TbCoreComponent @RequestMapping(TbUrlConstants.RPC_URL_PREFIX) @Slf4j public class RpcController extends BaseController { @@ -72,7 +69,7 @@ public class RpcController extends BaseController { protected final ObjectMapper jsonMapper = new ObjectMapper(); @Autowired - private DeviceRpcService deviceRpcService; + private TbCoreDeviceRpcService deviceRpcService; @Autowired private AccessValidator accessValidator; @@ -91,7 +88,6 @@ public class RpcController extends BaseController { return handleDeviceRPCRequest(false, new DeviceId(UUID.fromString(deviceIdStr)), requestBody); } - private DeferredResult handleDeviceRPCRequest(boolean oneWay, DeviceId deviceId, String requestBody) throws ThingsboardException { try { JsonNode rpcRequestBody = jsonMapper.readTree(requestBody); @@ -116,7 +112,7 @@ public class RpcController extends BaseController { timeout, body ); - deviceRpcService.processRestAPIRpcRequestToRuleEngine(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); + deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 5bfeeee523..0bc518e48c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -52,8 +52,10 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import org.thingsboard.server.service.security.permission.Operation; @@ -67,6 +69,7 @@ import java.util.stream.Collectors; @Slf4j @RestController +@TbCoreComponent @RequestMapping("/api") public class RuleChainController extends BaseController { @@ -130,7 +133,7 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - actorService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); logEntityAction(savedRuleChain.getId(), savedRuleChain, @@ -158,18 +161,18 @@ public class RuleChainController extends BaseController { TenantId tenantId = getCurrentUser().getTenantId(); RuleChain previousRootRuleChain = ruleChainService.getRootTenantRuleChain(tenantId); if (ruleChainService.setRootRuleChain(getTenantId(), ruleChainId)) { + if (previousRootRuleChain != null) { + previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - - actorService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), - ComponentLifecycleEvent.UPDATED); - - logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, - null, ActionType.UPDATED, null); + tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), + ComponentLifecycleEvent.UPDATED); + logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, + null, ActionType.UPDATED, null); + } ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, @@ -203,7 +206,7 @@ public class RuleChainController extends BaseController { RuleChain ruleChain = checkRuleChain(ruleChainMetaData.getRuleChainId(), Operation.WRITE); RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData)); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, null, @@ -254,9 +257,9 @@ public class RuleChainController extends BaseController { referencingRuleChainIds.remove(ruleChain.getId()); referencingRuleChainIds.forEach(referencingRuleChainId -> - actorService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); logEntityAction(ruleChainId, ruleChain, null, @@ -317,7 +320,7 @@ public class RuleChainController extends BaseController { ScriptEngine engine = null; try { engine = new RuleNodeJsScriptEngine(jsInvokeService, getCurrentUser().getId(), script, argNames); - TbMsg inMsg = new TbMsg(UUIDs.timeBased(), msgType, null, new TbMsgMetaData(metadata), data, null, null, 0L); + TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); switch (scriptType) { case "update": output = msgToOutput(engine.executeUpdate(inMsg)); diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index 43b525021e..bb866e3e9b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -15,12 +15,16 @@ */ package org.thingsboard.server.controller; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -56,16 +60,18 @@ import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -77,6 +83,7 @@ import org.thingsboard.server.service.telemetry.exception.UncheckedApiException; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -92,6 +99,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 22.03.18. */ @RestController +@TbCoreComponent @RequestMapping(TbUrlConstants.TELEMETRY_URL_PREFIX) @Slf4j public class TelemetryController extends BaseController { @@ -107,6 +115,8 @@ public class TelemetryController extends BaseController { private ExecutorService executor; + private static final ObjectMapper mapper = new ObjectMapper(); + @PostConstruct public void initExecutor() { executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("telemetry-controller")); @@ -166,7 +176,7 @@ public class TelemetryController extends BaseController { public DeferredResult getTimeseriesKeys( @PathVariable("entityType") String entityType, @PathVariable("entityId") String entityIdStr) throws ThingsboardException { return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr, - (result, tenantId, entityId) -> Futures.addCallback(tsService.findAllLatest(tenantId, entityId), getTsKeysToResponseCallback(result))); + (result, tenantId, entityId) -> Futures.addCallback(tsService.findAllLatest(tenantId, entityId), getTsKeysToResponseCallback(result), MoreExecutors.directExecutor())); } @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @@ -174,11 +184,12 @@ public class TelemetryController extends BaseController { @ResponseBody public DeferredResult getLatestTimeseries( @PathVariable("entityType") String entityType, @PathVariable("entityId") String entityIdStr, - @RequestParam(name = "keys", required = false) String keysStr) throws ThingsboardException { + @RequestParam(name = "keys", required = false) String keysStr, + @RequestParam(name = "useStrictDataTypes", required = false, defaultValue = "false") Boolean useStrictDataTypes) throws ThingsboardException { SecurityUser user = getCurrentUser(); return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr, - (result, tenantId, entityId) -> getLatestTimeseriesValuesCallback(result, user, entityId, keysStr)); + (result, tenantId, entityId) -> getLatestTimeseriesValuesCallback(result, user, entityId, keysStr, useStrictDataTypes)); } @@ -192,8 +203,8 @@ public class TelemetryController extends BaseController { @RequestParam(name = "endTs") Long endTs, @RequestParam(name = "interval", defaultValue = "0") Long interval, @RequestParam(name = "limit", defaultValue = "100") Integer limit, - @RequestParam(name = "agg", defaultValue = "NONE") String aggStr - ) throws ThingsboardException { + @RequestParam(name = "agg", defaultValue = "NONE") String aggStr, + @RequestParam(name = "useStrictDataTypes", required = false, defaultValue = "false") Boolean useStrictDataTypes) throws ThingsboardException { return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr, (result, tenantId, entityId) -> { // If interval is 0, convert this to a NONE aggregation, which is probably what the user really wanted @@ -201,7 +212,7 @@ public class TelemetryController extends BaseController { List queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, interval, limit, agg)) .collect(Collectors.toList()); - Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result)); + Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor()); }); } @@ -284,8 +295,7 @@ public class TelemetryController extends BaseController { if (startTs == null || endTs == null) { deleteToTs = endTs; return getImmediateDeferredResult("When deleteAllDataForKeys is false, start and end timestamp values shouldn't be empty", HttpStatus.BAD_REQUEST); - } - else{ + } else { deleteFromTs = startTs; deleteToTs = endTs; } @@ -334,7 +344,7 @@ public class TelemetryController extends BaseController { return deleteAttributes(entityId, scope, keysStr); } - private DeferredResult deleteAttributes(EntityId entityIdStr, String scope, String keysStr) throws ThingsboardException { + private DeferredResult deleteAttributes(EntityId entityIdSrc, String scope, String keysStr) throws ThingsboardException { List keys = toKeysList(keysStr); if (keys.isEmpty()) { return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST); @@ -344,19 +354,18 @@ public class TelemetryController extends BaseController { if (DataConstants.SERVER_SCOPE.equals(scope) || DataConstants.SHARED_SCOPE.equals(scope) || DataConstants.CLIENT_SCOPE.equals(scope)) { - return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdStr, (result, tenantId, entityId) -> { + return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { ListenableFuture> future = attributesService.removeAll(user.getTenantId(), entityId, scope, keys); Futures.addCallback(future, new FutureCallback>() { @Override public void onSuccess(@Nullable List tmp) { logAttributesDeleted(user, entityId, scope, keys, null); - if (entityId.getEntityType() == EntityType.DEVICE) { + if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onDelete( - user.getTenantId(), deviceId, keysToNotify); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete( + user.getTenantId(), deviceId, keysToNotify), null); } result.setResult(new ResponseEntity<>(HttpStatus.OK)); } @@ -388,12 +397,6 @@ public class TelemetryController extends BaseController { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); - if (entityId.getEntityType() == EntityType.DEVICE) { - DeviceId deviceId = new DeviceId(entityId.getId()); - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate( - user.getTenantId(), deviceId, scope, attributes); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); - } result.setResult(new ResponseEntity(HttpStatus.OK)); } @@ -447,14 +450,14 @@ public class TelemetryController extends BaseController { }); } - private void getLatestTimeseriesValuesCallback(@Nullable DeferredResult result, SecurityUser user, EntityId entityId, String keys) { + private void getLatestTimeseriesValuesCallback(@Nullable DeferredResult result, SecurityUser user, EntityId entityId, String keys, Boolean useStrictDataTypes) { ListenableFuture> future; if (StringUtils.isEmpty(keys)) { future = tsService.findAllLatest(user.getTenantId(), entityId); } else { future = tsService.findLatest(user.getTenantId(), entityId, toKeysList(keys)); } - Futures.addCallback(future, getTsKvListCallback(result)); + Futures.addCallback(future, getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor()); } private void getAttributeValuesCallback(@Nullable DeferredResult result, SecurityUser user, EntityId entityId, String scope, String keys) { @@ -462,9 +465,9 @@ public class TelemetryController extends BaseController { FutureCallback> callback = getAttributeValuesToResponseCallback(result, user, scope, entityId, keyList); if (!StringUtils.isEmpty(scope)) { if (keyList != null && !keyList.isEmpty()) { - Futures.addCallback(attributesService.find(user.getTenantId(), entityId, scope, keyList), callback); + Futures.addCallback(attributesService.find(user.getTenantId(), entityId, scope, keyList), callback, MoreExecutors.directExecutor()); } else { - Futures.addCallback(attributesService.findAll(user.getTenantId(), entityId, scope), callback); + Futures.addCallback(attributesService.findAll(user.getTenantId(), entityId, scope), callback, MoreExecutors.directExecutor()); } } else { List>> futures = new ArrayList<>(); @@ -478,12 +481,12 @@ public class TelemetryController extends BaseController { ListenableFuture> future = mergeAllAttributesFutures(futures); - Futures.addCallback(future, callback); + Futures.addCallback(future, callback, MoreExecutors.directExecutor()); } } private void getAttributeKeysCallback(@Nullable DeferredResult result, TenantId tenantId, EntityId entityId, String scope) { - Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), getAttributeKeysToResponseCallback(result)); + Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), getAttributeKeysToResponseCallback(result), MoreExecutors.directExecutor()); } private void getAttributeKeysCallback(@Nullable DeferredResult result, TenantId tenantId, EntityId entityId) { @@ -494,7 +497,7 @@ public class TelemetryController extends BaseController { ListenableFuture> future = mergeAllAttributesFutures(futures); - Futures.addCallback(future, getAttributeKeysToResponseCallback(result)); + Futures.addCallback(future, getAttributeKeysToResponseCallback(result), MoreExecutors.directExecutor()); } private FutureCallback> getTsKeysToResponseCallback(final DeferredResult response) { @@ -536,8 +539,9 @@ public class TelemetryController extends BaseController { return new FutureCallback>() { @Override public void onSuccess(List attributes) { - List values = attributes.stream().map(attribute -> new AttributeData(attribute.getLastUpdateTs(), - attribute.getKey(), attribute.getValue())).collect(Collectors.toList()); + List values = attributes.stream().map(attribute -> + new AttributeData(attribute.getLastUpdateTs(), attribute.getKey(), getKvValue(attribute)) + ).collect(Collectors.toList()); logAttributesRead(user, entityId, scope, keyList, null); response.setResult(new ResponseEntity<>(values, HttpStatus.OK)); } @@ -551,14 +555,14 @@ public class TelemetryController extends BaseController { }; } - private FutureCallback> getTsKvListCallback(final DeferredResult response) { + private FutureCallback> getTsKvListCallback(final DeferredResult response, Boolean useStrictDataTypes) { return new FutureCallback>() { @Override public void onSuccess(List data) { Map> result = new LinkedHashMap<>(); for (TsKvEntry entry : data) { - result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()) - .add(new TsData(entry.getTs(), entry.getValueAsString())); + Object value = useStrictDataTypes ? getKvValue(entry) : entry.getValueAsString(); + result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(new TsData(entry.getTs(), value)); } response.setResult(new ResponseEntity<>(result, HttpStatus.OK)); } @@ -639,7 +643,9 @@ public class TelemetryController extends BaseController { jsonNode.fields().forEachRemaining(entry -> { String key = entry.getKey(); JsonNode value = entry.getValue(); - if (entry.getValue().isTextual()) { + if (entry.getValue().isObject() || entry.getValue().isArray()) { + attributes.add(new BaseAttributeKvEntry(new JsonDataEntry(key, toJsonStr(value)), ts)); + } else if (entry.getValue().isTextual()) { if (maxStringValueLength > 0 && entry.getValue().textValue().length() > maxStringValueLength) { String message = String.format("String value length [%d] for key [%s] is greater than maximum allowed [%d]", entry.getValue().textValue().length(), key, maxStringValueLength); throw new UncheckedApiException(new InvalidParametersException(message)); @@ -659,4 +665,27 @@ public class TelemetryController extends BaseController { }); return attributes; } + + private String toJsonStr(JsonNode value) { + try { + return mapper.writeValueAsString(value); + } catch (JsonProcessingException e) { + throw new JsonParseException("Can't parse jsonValue: " + value, e); + } + } + + private JsonNode toJsonNode(String value) { + try { + return mapper.readTree(value); + } catch (IOException e) { + throw new JsonParseException("Can't parse jsonValue: " + value, e); + } + } + + private Object getKvValue(KvEntry entry) { + if (entry.getDataType() == DataType.JSON) { + return toJsonNode(entry.getJsonValue().get()); + } + return entry.getValue(); + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 8d95e78cfa..9def943e88 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -34,11 +34,13 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController +@TbCoreComponent @RequestMapping("/api") @Slf4j public class TenantController extends BaseController { @@ -74,7 +76,6 @@ public class TenantController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.TENANT, operation, tenant.getId(), tenant); - tenant = checkNotNull(tenantService.saveTenant(tenant)); if (newTenant) { installScripts.createDefaultRuleChains(tenant.getId()); @@ -94,8 +95,7 @@ public class TenantController extends BaseController { TenantId tenantId = new TenantId(toUUID(strTenantId)); checkTenantId(tenantId, Operation.DELETE); tenantService.deleteTenant(tenantId); - - actorService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); + tbClusterService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index a24efdb309..bf64ad7d6f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -51,10 +52,12 @@ import org.thingsboard.server.service.security.model.token.JwtToken; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import org.thingsboard.server.utils.MiscUtils; import javax.servlet.http.HttpServletRequest; @RestController +@TbCoreComponent @RequestMapping("/api") public class UserController extends BaseController { @@ -146,7 +149,7 @@ public class UserController extends BaseController { if (sendEmail) { SecurityUser authUser = getCurrentUser(); UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), savedUser.getId()); - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, userCredentials.getActivateToken()); String email = savedUser.getEmail(); @@ -186,7 +189,7 @@ public class UserController extends BaseController { UserCredentials userCredentials = userService.findUserCredentialsByUserId(getCurrentUser().getTenantId(), user.getId()); if (!userCredentials.isEnabled()) { - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, userCredentials.getActivateToken()); mailService.sendActivationEmail(activateUrl, email); @@ -211,7 +214,7 @@ public class UserController extends BaseController { SecurityUser authUser = getCurrentUser(); UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), user.getId()); if (!userCredentials.isEnabled()) { - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, userCredentials.getActivateToken()); return activateUrl; diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java index ce3e65c0a1..debe49b018 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java @@ -31,12 +31,14 @@ import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController +@TbCoreComponent @RequestMapping("/api") public class WidgetTypeController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java index b76306e071..3d5cd22400 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java @@ -32,13 +32,14 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController +@TbCoreComponent @RequestMapping("/api") public class WidgetsBundleController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index f2862c5ab6..6e33133522 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.config.WebSocketConfiguration; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.telemetry.SessionEvent; @@ -52,6 +53,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; @Service +@TbCoreComponent @Slf4j public class TbWebSocketHandler extends TextWebSocketHandler implements TelemetryWebSocketMsgEndpoint { @@ -63,7 +65,6 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr @Value("${server.ws.send_timeout:5000}") private long sendTimeout; - @Value("${server.ws.limits.max_sessions_per_tenant:0}") private int maxSessionsPerTenant; @Value("${server.ws.limits.max_sessions_per_customer:0}") @@ -91,10 +92,10 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr try { SessionMetaData sessionMd = internalSessionMap.get(session.getId()); if (sessionMd != null) { - log.info("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); + log.trace("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); webSocketService.handleWebSocketMsg(sessionMd.sessionRef, message.getPayload()); } else { - log.warn("[{}] Failed to find session", session.getId()); + log.trace("[{}] Failed to find session", session.getId()); session.close(CloseStatus.SERVER_ERROR.withReason("Session not found!")); } } catch (IOException e) { @@ -138,7 +139,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr if (sessionMd != null) { processInWebSocketService(sessionMd.sessionRef, SessionEvent.onError(tError)); } else { - log.warn("[{}] Failed to find session", session.getId()); + log.trace("[{}] Failed to find session", session.getId()); } log.trace("[{}] Session transport error", session.getId(), tError); } diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index f424e262eb..8dd45bc41b 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -23,7 +23,8 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.service.component.ComponentDiscoveryService; -import org.thingsboard.server.service.install.DatabaseUpgradeService; +import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; +import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.TsDatabaseSchemaService; @@ -50,7 +51,10 @@ public class ThingsboardInstallService { private TsDatabaseSchemaService tsDatabaseSchemaService; @Autowired - private DatabaseUpgradeService databaseUpgradeService; + private DatabaseEntitiesUpgradeService databaseEntitiesUpgradeService; + + @Autowired(required = false) + private DatabaseTsUpgradeService databaseTsUpgradeService; @Autowired private ComponentDiscoveryService componentDiscoveryService; @@ -73,48 +77,48 @@ public class ThingsboardInstallService { case "1.2.3": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion log.info("Upgrading ThingsBoard from version 1.2.3 to 1.3.0 ..."); - databaseUpgradeService.upgradeDatabase("1.2.3"); + databaseEntitiesUpgradeService.upgradeDatabase("1.2.3"); case "1.3.0": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion log.info("Upgrading ThingsBoard from version 1.3.0 to 1.3.1 ..."); - databaseUpgradeService.upgradeDatabase("1.3.0"); + databaseEntitiesUpgradeService.upgradeDatabase("1.3.0"); case "1.3.1": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion log.info("Upgrading ThingsBoard from version 1.3.1 to 1.4.0 ..."); - databaseUpgradeService.upgradeDatabase("1.3.1"); + databaseEntitiesUpgradeService.upgradeDatabase("1.3.1"); case "1.4.0": log.info("Upgrading ThingsBoard from version 1.4.0 to 2.0.0 ..."); - databaseUpgradeService.upgradeDatabase("1.4.0"); + databaseEntitiesUpgradeService.upgradeDatabase("1.4.0"); dataUpdateService.updateData("1.4.0"); case "2.0.0": log.info("Upgrading ThingsBoard from version 2.0.0 to 2.1.1 ..."); - databaseUpgradeService.upgradeDatabase("2.0.0"); + databaseEntitiesUpgradeService.upgradeDatabase("2.0.0"); case "2.1.1": log.info("Upgrading ThingsBoard from version 2.1.1 to 2.1.2 ..."); - databaseUpgradeService.upgradeDatabase("2.1.1"); + databaseEntitiesUpgradeService.upgradeDatabase("2.1.1"); case "2.1.3": log.info("Upgrading ThingsBoard from version 2.1.3 to 2.2.0 ..."); - databaseUpgradeService.upgradeDatabase("2.1.3"); + databaseEntitiesUpgradeService.upgradeDatabase("2.1.3"); case "2.3.0": log.info("Upgrading ThingsBoard from version 2.3.0 to 2.3.1 ..."); - databaseUpgradeService.upgradeDatabase("2.3.0"); + databaseEntitiesUpgradeService.upgradeDatabase("2.3.0"); case "2.3.1": log.info("Upgrading ThingsBoard from version 2.3.1 to 2.4.0 ..."); - databaseUpgradeService.upgradeDatabase("2.3.1"); + databaseEntitiesUpgradeService.upgradeDatabase("2.3.1"); case "2.4.0": log.info("Upgrading ThingsBoard from version 2.4.0 to 2.4.1 ..."); @@ -122,11 +126,19 @@ public class ThingsboardInstallService { case "2.4.1": log.info("Upgrading ThingsBoard from version 2.4.1 to 2.4.2 ..."); - databaseUpgradeService.upgradeDatabase("2.4.1"); + databaseEntitiesUpgradeService.upgradeDatabase("2.4.1"); case "2.4.2": log.info("Upgrading ThingsBoard from version 2.4.2 to 2.4.3 ..."); - databaseUpgradeService.upgradeDatabase("2.4.2"); + databaseEntitiesUpgradeService.upgradeDatabase("2.4.2"); + + case "2.4.3": + log.info("Upgrading ThingsBoard from version 2.4.3 to 2.5 ..."); + + if (databaseTsUpgradeService != null) { + databaseTsUpgradeService.upgradeDatabase("2.4.3"); + } + databaseEntitiesUpgradeService.upgradeDatabase("2.4.3"); log.info("Updating system data..."); diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java deleted file mode 100644 index 9a7c83f167..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; - -import javax.annotation.PostConstruct; - -import static org.thingsboard.server.utils.MiscUtils.missingProperty; - -/** - * @author Andrew Shvayka - */ -@Service -@Slf4j -public class CurrentServerInstanceService implements ServerInstanceService { - - @Value("${rpc.bind_host}") - private String rpcHost; - @Value("${rpc.bind_port}") - private Integer rpcPort; - - private ServerInstance self; - - @PostConstruct - public void init() { - Assert.hasLength(rpcHost, missingProperty("rpc.bind_host")); - Assert.notNull(rpcPort, missingProperty("rpc.bind_port")); - self = new ServerInstance(new ServerAddress(rpcHost, rpcPort, ServerType.CORE)); - log.info("Current server instance: [{};{}]", self.getHost(), self.getPort()); - } - - @Override - public ServerInstance getSelf() { - return self; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java deleted file mode 100644 index e9325800af..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@ToString -@EqualsAndHashCode(exclude = {"serverInfo", "serverAddress"}) -public final class ServerInstance implements Comparable { - - @Getter - private final String host; - @Getter - private final int port; - @Getter - private final ServerAddress serverAddress; - - public ServerInstance(ServerAddress serverAddress) { - this.serverAddress = serverAddress; - this.host = serverAddress.getHost(); - this.port = serverAddress.getPort(); - } - - @Override - public int compareTo(ServerInstance o) { - return this.serverAddress.compareTo(o.serverAddress); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java deleted file mode 100644 index e0ed64fdbd..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; - -import java.util.Optional; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -public interface ClusterRoutingService extends DiscoveryServiceListener { - - ServerAddress getCurrentServer(); - - Optional resolveById(EntityId entityId); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java deleted file mode 100644 index e114a9b6ab..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import com.google.common.hash.HashCode; -import com.google.common.hash.HashFunction; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.utils.MiscUtils; - -import javax.annotation.PostConstruct; -import java.util.Arrays; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -/** - * Cluster service implementation based on consistent hash ring - */ - -@Service -@Slf4j -public class ConsistentClusterRoutingService implements ClusterRoutingService { - - @Autowired - private DiscoveryService discoveryService; - - @Value("${cluster.hash_function_name}") - private String hashFunctionName; - @Value("${cluster.vitrual_nodes_size}") - private Integer virtualNodesSize; - - private ServerInstance currentServer; - - private HashFunction hashFunction; - - private ConsistentHashCircle[] circles; - private ConsistentHashCircle rootCircle; - - @PostConstruct - public void init() { - log.info("Initializing Cluster routing service!"); - this.hashFunction = MiscUtils.forName(hashFunctionName); - this.currentServer = discoveryService.getCurrentServer(); - this.circles = new ConsistentHashCircle[ServerType.values().length]; - for (ServerType serverType : ServerType.values()) { - circles[serverType.ordinal()] = new ConsistentHashCircle(); - } - rootCircle = circles[ServerType.CORE.ordinal()]; - addNode(discoveryService.getCurrentServer()); - for (ServerInstance instance : discoveryService.getOtherServers()) { - addNode(instance); - } - logCircle(); - log.info("Cluster routing service initialized!"); - } - - @Override - public ServerAddress getCurrentServer() { - return discoveryService.getCurrentServer().getServerAddress(); - } - - @Override - public Optional resolveById(EntityId entityId) { - return resolveByUuid(rootCircle, entityId.getId()); - } - - private Optional resolveByUuid(ConsistentHashCircle circle, UUID uuid) { - Assert.notNull(uuid); - if (circle.isEmpty()) { - return Optional.empty(); - } - Long hash = hashFunction.newHasher().putLong(uuid.getMostSignificantBits()) - .putLong(uuid.getLeastSignificantBits()).hash().asLong(); - if (!circle.containsKey(hash)) { - ConcurrentNavigableMap tailMap = - circle.tailMap(hash); - hash = tailMap.isEmpty() ? - circle.firstKey() : tailMap.firstKey(); - } - ServerInstance result = circle.get(hash); - if (!currentServer.equals(result)) { - return Optional.of(result.getServerAddress()); - } else { - return Optional.empty(); - } - } - - @Override - public void onServerAdded(ServerInstance server) { - log.info("On server added event: {}", server); - addNode(server); - logCircle(); - } - - @Override - public void onServerUpdated(ServerInstance server) { - log.debug("Ignoring server onUpdate event: {}", server); - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.info("On server removed event: {}", server); - removeNode(server); - logCircle(); - } - - private void addNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].put(hash(instance, i).asLong(), instance); - } - } - - private void removeNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].remove(hash(instance, i).asLong()); - } - } - - private HashCode hash(ServerInstance instance, int i) { - return hashFunction.newHasher().putString(instance.getHost(), MiscUtils.UTF8).putInt(instance.getPort()).putInt(i).hash(); - } - - private void logCircle() { - log.trace("Consistent Hash Circle Start"); - Arrays.asList(circles).forEach(ConsistentHashCircle::log); - log.trace("Consistent Hash Circle End"); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java deleted file mode 100644 index 23b050634c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import com.google.protobuf.ByteString; -import io.grpc.Server; -import io.grpc.ServerBuilder; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.service.cluster.discovery.ServerInstanceService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; - -import javax.annotation.PreDestroy; -import java.io.IOException; -import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -/** - * @author Andrew Shvayka - */ -@Service -@Slf4j -public class ClusterGrpcService extends ClusterRpcServiceGrpc.ClusterRpcServiceImplBase implements ClusterRpcService { - - @Autowired - private ServerInstanceService instanceService; - - @Autowired - private DataDecodingEncodingService encodingService; - - private RpcMsgListener listener; - - private Server server; - - private ServerInstance instance; - - private ConcurrentMap>> pendingSessionMap = - new ConcurrentHashMap<>(); - - public void init(RpcMsgListener listener) { - this.listener = listener; - log.info("Initializing RPC service!"); - instance = instanceService.getSelf(); - server = ServerBuilder.forPort(instance.getPort()).addService(this).build(); - log.info("Going to start RPC server using port: {}", instance.getPort()); - try { - server.start(); - } catch (IOException e) { - log.error("Failed to start RPC server!", e); - throw new RuntimeException("Failed to start RPC server!"); - } - log.info("RPC service initialized!"); - } - - @Override - public void onSessionCreated(UUID msgUid, StreamObserver inputStream) { - BlockingQueue> queue = pendingSessionMap.remove(msgUid); - if (queue != null) { - try { - queue.put(inputStream); - } catch (InterruptedException e) { - log.warn("Failed to report created session!"); - Thread.currentThread().interrupt(); - } - } else { - log.warn("Failed to lookup pending session!"); - } - } - - @Override - public StreamObserver handleMsgs( - StreamObserver responseObserver) { - log.info("Processing new session."); - return createSession(new RpcSessionCreateRequestMsg(UUID.randomUUID(), null, responseObserver)); - } - - - @PreDestroy - public void stop() { - if (server != null) { - log.info("Going to onStop RPC server"); - server.shutdownNow(); - try { - server.awaitTermination(); - log.info("RPC server stopped!"); - } catch (InterruptedException e) { - log.warn("Failed to onStop RPC server!"); - Thread.currentThread().interrupt(); - } - } - } - - - @Override - public void broadcast(RpcBroadcastMsg msg) { - listener.onBroadcastMsg(msg); - } - - private StreamObserver createSession(RpcSessionCreateRequestMsg msg) { - BlockingQueue> queue = new ArrayBlockingQueue<>(1); - pendingSessionMap.put(msg.getMsgUid(), queue); - listener.onRpcSessionCreateRequestMsg(msg); - try { - StreamObserver observer = queue.take(); - log.info("Processed new session."); - return observer; - } catch (Exception e) { - log.info("Failed to process session.", e); - throw new RuntimeException(e); - } - } - - @Override - public void tell(ClusterAPIProtos.ClusterMessage message) { - listener.onSendMsg(message); - } - - @Override - public void tell(ServerAddress serverAddress, TbActorMsg actorMsg) { - listener.onSendMsg(encodingService.convertToProtoDataMessage(serverAddress, actorMsg)); - } - - @Override - public void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data) { - ClusterAPIProtos.ClusterMessage msg = ClusterAPIProtos.ClusterMessage - .newBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(serverAddress.getHost()) - .setPort(serverAddress.getPort()) - .build()) - .setMessageType(msgType) - .setPayload(ByteString.copyFrom(data)).build(); - listener.onSendMsg(msg); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java deleted file mode 100644 index 637924fca9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import io.grpc.stub.StreamObserver; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -public interface ClusterRpcService { - - void init(RpcMsgListener listener); - - void broadcast(RpcBroadcastMsg msg); - - void onSessionCreated(UUID msgUid, StreamObserver inputStream); - - void tell(ClusterAPIProtos.ClusterMessage message); - - void tell(ServerAddress serverAddress, TbActorMsg actorMsg); - - void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java deleted file mode 100644 index aff0bc4a41..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import io.grpc.Channel; -import io.grpc.ManagedChannel; -import io.grpc.stub.StreamObserver; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.io.Closeable; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -@Slf4j -public final class GrpcSession implements Closeable { - private final UUID sessionId; - private final boolean client; - private final GrpcSessionListener listener; - private final ManagedChannel channel; - private StreamObserver inputStream; - private StreamObserver outputStream; - - private boolean connected; - private ServerAddress remoteServer; - - public GrpcSession(GrpcSessionListener listener) { - this(null, listener, null); - } - - public GrpcSession(ServerAddress remoteServer, GrpcSessionListener listener, ManagedChannel channel) { - this.sessionId = UUID.randomUUID(); - this.listener = listener; - if (remoteServer != null) { - this.client = true; - this.connected = true; - this.remoteServer = remoteServer; - } else { - this.client = false; - } - this.channel = channel; - } - - public void initInputStream() { - this.inputStream = new StreamObserver() { - @Override - public void onNext(ClusterAPIProtos.ClusterMessage clusterMessage) { - if (!connected && clusterMessage.getMessageType() == ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE) { - connected = true; - ServerAddress rpcAddress = new ServerAddress(clusterMessage.getServerAddress().getHost(), clusterMessage.getServerAddress().getPort(), ServerType.CORE); - remoteServer = new ServerAddress(rpcAddress.getHost(), rpcAddress.getPort(), ServerType.CORE); - listener.onConnected(GrpcSession.this); - } - if (connected) { - listener.onReceiveClusterGrpcMsg(GrpcSession.this, clusterMessage); - } - } - - @Override - public void onError(Throwable t) { - listener.onError(GrpcSession.this, t); - } - - @Override - public void onCompleted() { - outputStream.onCompleted(); - listener.onDisconnected(GrpcSession.this); - } - }; - } - - public void initOutputStream() { - if (client) { - listener.onConnected(GrpcSession.this); - } - } - - public void sendMsg(ClusterAPIProtos.ClusterMessage msg) { - if (connected) { - try { - outputStream.onNext(msg); - } catch (Throwable t) { - try { - outputStream.onError(t); - } catch (Throwable t2) { - } - listener.onError(GrpcSession.this, t); - } - } else { - log.warn("[{}] Failed to send message due to closed session!", sessionId); - } - } - - @Override - public void close() { - connected = false; - try { - outputStream.onCompleted(); - } catch (IllegalStateException e) { - log.debug("[{}] Failed to close output stream: {}", sessionId, e.getMessage()); - } - if (channel != null) { - channel.shutdownNow(); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java deleted file mode 100644 index 2ff37b39e0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ - -public interface RpcMsgListener { - void onReceivedMsg(ServerAddress remoteServer, ClusterAPIProtos.ClusterMessage msg); - void onSendMsg(ClusterAPIProtos.ClusterMessage msg); - void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg); - void onBroadcastMsg(RpcBroadcastMsg msg); -} diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java index a5d3ab465e..4a781b8673 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java @@ -16,8 +16,6 @@ package org.thingsboard.server.service.encoding; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; @@ -27,8 +25,5 @@ public interface DataDecodingEncodingService { byte[] encode(TbActorMsg msq); - ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, - TbActorMsg msg); - } diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java index 45bb9f78fd..8d89059488 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java @@ -15,25 +15,19 @@ */ package org.thingsboard.server.service.encoding; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; - - @Slf4j @Service public class ProtoWithFSTService implements DataDecodingEncodingService { - private final FSTConfiguration config = FSTConfiguration.createDefaultConfiguration(); + @Override public Optional decode(byte[] byteArray) { try { @@ -42,7 +36,7 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { } catch (IllegalArgumentException e) { log.error("Error during deserialization message, [{}]", e.getMessage()); - return Optional.empty(); + return Optional.empty(); } } @@ -51,18 +45,4 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { return config.asByteArray(msq); } - @Override - public ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, - TbActorMsg msg) { - return ClusterAPIProtos.ClusterMessage - .newBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(serverAddress.getHost()) - .setPort(serverAddress.getPort()) - .build()) - .setMessageType(CLUSTER_ACTOR_MESSAGE) - .setPayload(ByteString.copyFrom(encode(msg))).build(); - - } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/AbstractCassandraDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/AbstractCassandraDatabaseUpgradeService.java new file mode 100644 index 0000000000..603158314a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/AbstractCassandraDatabaseUpgradeService.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cassandra.CassandraInstallCluster; +import org.thingsboard.server.service.install.cql.CQLStatementsParser; + +import java.nio.file.Path; +import java.util.List; + +@Slf4j +public abstract class AbstractCassandraDatabaseUpgradeService { + @Autowired + protected CassandraCluster cluster; + + @Autowired + @Qualifier("CassandraInstallCluster") + private CassandraInstallCluster installCluster; + + protected void loadCql(Path cql) throws Exception { + List statements = new CQLStatementsParser(cql).getStatements(); + statements.forEach(statement -> { + installCluster.getSession().execute(statement); + try { + Thread.sleep(2500); + } catch (InterruptedException e) { + } + }); + Thread.sleep(5000); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java new file mode 100644 index 0000000000..7afa422460 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java @@ -0,0 +1,114 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.Statement; + +@Slf4j +public abstract class AbstractSqlTsDatabaseUpgradeService { + + protected static final String CALL_REGEX = "call "; + protected static final String DROP_TABLE = "DROP TABLE "; + protected static final String DROP_PROCEDURE_IF_EXISTS = "DROP PROCEDURE IF EXISTS "; + + @Value("${spring.datasource.url}") + protected String dbUrl; + + @Value("${spring.datasource.username}") + protected String dbUserName; + + @Value("${spring.datasource.password}") + protected String dbPassword; + + @Autowired + protected InstallScripts installScripts; + + protected abstract void loadSql(Connection conn, String fileName); + + protected void loadFunctions(Path sqlFile, Connection conn) throws Exception { + String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8); + conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script + } + + protected boolean checkVersion(Connection conn) { + boolean versionValid = false; + try { + Statement statement = conn.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT current_setting('server_version_num')"); + resultSet.next(); + if(resultSet.getLong(1) > 110000) { + versionValid = true; + } + statement.close(); + } catch (Exception e) { + log.info("Failed to check current PostgreSQL version due to: {}", e.getMessage()); + } + return versionValid; + } + + protected boolean isOldSchema(Connection conn, long fromVersion) { + boolean isOldSchema = true; + try { + Statement statement = conn.createStatement(); + statement.execute("CREATE TABLE IF NOT EXISTS tb_schema_settings ( schema_version bigint NOT NULL, CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version));"); + Thread.sleep(1000); + ResultSet resultSet = statement.executeQuery("SELECT schema_version FROM tb_schema_settings;"); + if (resultSet.next()) { + isOldSchema = resultSet.getLong(1) <= fromVersion; + } else { + resultSet.close(); + statement.execute("INSERT INTO tb_schema_settings (schema_version) VALUES (" + fromVersion + ")"); + } + statement.close(); + } catch (InterruptedException | SQLException e) { + log.info("Failed to check current PostgreSQL schema due to: {}", e.getMessage()); + } + return isOldSchema; + } + + protected void executeQuery(Connection conn, String query) { + try { + Statement statement = conn.createStatement(); + statement.execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script + SQLWarning warnings = statement.getWarnings(); + if (warnings != null) { + log.info("{}", warnings.getMessage()); + SQLWarning nextWarning = warnings.getNextWarning(); + while (nextWarning != null) { + log.info("{}", nextWarning.getMessage()); + nextWarning = nextWarning.getNextWarning(); + } + } + Thread.sleep(2000); + log.info("Successfully executed query: {}", query); + } catch (InterruptedException | SQLException e) { + log.error("Failed to execute query: {} due to: {}", query, e.getMessage()); + throw new RuntimeException("Failed to execute query:" + query + " due to: ", e); + } + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java index 285a191c04..573d152635 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java @@ -19,20 +19,15 @@ import com.datastax.driver.core.KeyspaceMetadata; import com.datastax.driver.core.exceptions.InvalidQueryException; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; -import org.thingsboard.server.dao.cassandra.CassandraCluster; -import org.thingsboard.server.dao.cassandra.CassandraInstallCluster; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.util.NoSqlDao; -import org.thingsboard.server.service.install.cql.CQLStatementsParser; import org.thingsboard.server.service.install.cql.CassandraDbHelper; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; import static org.thingsboard.server.service.install.DatabaseHelper.ASSET; @@ -59,17 +54,10 @@ import static org.thingsboard.server.service.install.DatabaseHelper.TYPE; @NoSqlDao @Profile("install") @Slf4j -public class CassandraDatabaseUpgradeService implements DatabaseUpgradeService { +public class CassandraDatabaseUpgradeService extends AbstractCassandraDatabaseUpgradeService implements DatabaseEntitiesUpgradeService { private static final String SCHEMA_UPDATE_CQL = "schema_update.cql"; - @Autowired - private CassandraCluster cluster; - - @Autowired - @Qualifier("CassandraInstallCluster") - private CassandraInstallCluster installCluster; - @Autowired private DashboardService dashboardService; @@ -264,7 +252,8 @@ public class CassandraDatabaseUpgradeService implements DatabaseUpgradeService { try { cluster.getSession().execute(updateDeviceTableStmt); Thread.sleep(2500); - } catch (InvalidQueryException e) {} + } catch (InvalidQueryException e) { + } log.info("Schema updated."); break; case "2.4.1": @@ -275,7 +264,8 @@ public class CassandraDatabaseUpgradeService implements DatabaseUpgradeService { cluster.getSession().execute(updateAssetTableStmt); Thread.sleep(2500); log.info("Assets updated."); - } catch (InvalidQueryException e) {} + } catch (InvalidQueryException e) { + } log.info("Schema updated."); break; case "2.4.2": @@ -286,24 +276,39 @@ public class CassandraDatabaseUpgradeService implements DatabaseUpgradeService { cluster.getSession().execute(updateAlarmTableStmt); Thread.sleep(2500); log.info("Alarms updated."); - } catch (InvalidQueryException e) {} + } catch (InvalidQueryException e) { + } + log.info("Schema updated."); + break; + case "2.4.3": + log.info("Updating schema ..."); + String updateAttributeKvTableStmt = "alter table attributes_kv_cf add json_v text"; + try { + log.info("Updating attributes ..."); + cluster.getSession().execute(updateAttributeKvTableStmt); + Thread.sleep(2500); + log.info("Attributes updated."); + } catch (InvalidQueryException e) { + } + + String updateTenantCoreTableStmt = "alter table tenant add isolated_tb_core boolean"; + String updateTenantRuleEngineTableStmt = "alter table tenant add isolated_tb_rule_engine boolean"; + + try { + log.info("Updating tenant..."); + cluster.getSession().execute(updateTenantCoreTableStmt); + Thread.sleep(2500); + + cluster.getSession().execute(updateTenantRuleEngineTableStmt); + Thread.sleep(2500); + log.info("Tenant updated."); + } catch (InvalidQueryException e) { + } log.info("Schema updated."); break; default: throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); } - - } - - private void loadCql(Path cql) throws Exception { - List statements = new CQLStatementsParser(cql).getStatements(); - statements.forEach(statement -> { - installCluster.getSession().execute(statement); - try { - Thread.sleep(2500); - } catch (InterruptedException e) {} - }); - Thread.sleep(5000); } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java new file mode 100644 index 0000000000..4bc68e92bc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import com.datastax.driver.core.exceptions.InvalidQueryException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.NoSqlTsDao; + +@Service +@NoSqlTsDao +@Profile("install") +@Slf4j +public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabaseUpgradeService implements DatabaseTsUpgradeService { + + @Override + public void upgradeDatabase(String fromVersion) throws Exception { + switch (fromVersion) { + case "2.4.3": + log.info("Updating schema ..."); + String updateTsKvTableStmt = "alter table ts_kv_cf add json_v text"; + String updateTsKvLatestTableStmt = "alter table ts_kv_latest_cf add json_v text"; + + try { + log.info("Updating ts ..."); + cluster.getSession().execute(updateTsKvTableStmt); + Thread.sleep(2500); + log.info("Ts updated."); + log.info("Updating ts latest ..."); + cluster.getSession().execute(updateTsKvLatestTableStmt); + Thread.sleep(2500); + log.info("Ts latest updated."); + } catch (InvalidQueryException e) { + } + log.info("Schema updated."); + break; + default: + throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/install/DatabaseEntitiesUpgradeService.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java rename to application/src/main/java/org/thingsboard/server/service/install/DatabaseEntitiesUpgradeService.java index 095e10f0b3..7abb97a6a9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DatabaseEntitiesUpgradeService.java @@ -13,12 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.discovery; +package org.thingsboard.server.service.install; -/** - * @author Andrew Shvayka - */ -public interface ServerInstanceService { +public interface DatabaseEntitiesUpgradeService { + + void upgradeDatabase(String fromVersion) throws Exception; - ServerInstance getSelf(); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/DatabaseTsUpgradeService.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java rename to application/src/main/java/org/thingsboard/server/service/install/DatabaseTsUpgradeService.java index 47a3944e74..fe82cabff9 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DatabaseTsUpgradeService.java @@ -15,8 +15,8 @@ */ package org.thingsboard.server.service.install; -public interface DatabaseUpgradeService { +public interface DatabaseTsUpgradeService { void upgradeDatabase(String fromVersion) throws Exception; -} +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index beba984077..329dc37d3f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -25,24 +25,36 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; +import java.util.Arrays; + @Service @Profile("install") @Slf4j @@ -73,9 +85,18 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Autowired private CustomerService customerService; + @Autowired + private RelationService relationService; + + @Autowired + private AssetService assetService; + @Autowired private DeviceService deviceService; + @Autowired + private AttributesService attributesService; + @Autowired private DeviceCredentialsService deviceCredentialsService; @@ -106,9 +127,10 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { node.put("smtpHost", "localhost"); node.put("smtpPort", "25"); node.put("timeout", "10000"); - node.put("enableTls", "false"); + node.put("enableTls", false); node.put("username", ""); - node.put("password", ""); //NOSONAR, key used to identify password field (not password value itself) + node.put("password", ""); + node.put("tlsVersion", "TLSv1.2");//NOSONAR, key used to identify password field (not password value itself) mailSettings.setJsonValue(node); adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, mailSettings); } @@ -119,7 +141,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { demoTenant.setRegion("Global"); demoTenant.setTitle("Tenant"); demoTenant = tenantService.saveTenant(demoTenant); - installScripts.createDefaultRuleChains(demoTenant.getId()); + installScripts.loadDemoRuleChains(demoTenant.getId()); createUser(Authority.TENANT_ADMIN, demoTenant.getId(), null, "tenant@thingsboard.org", "tenant"); Customer customerA = new Customer(); @@ -151,6 +173,34 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { createDevice(demoTenant.getId(), null, DEFAULT_DEVICE_TYPE, "Raspberry Pi Demo Device", "RASPBERRY_PI_DEMO_TOKEN", "Demo device that is used in " + "Raspberry Pi GPIO control sample application"); + Asset thermostatAlarms = new Asset(); + thermostatAlarms.setTenantId(demoTenant.getId()); + thermostatAlarms.setName("Thermostat Alarms"); + thermostatAlarms.setType("AlarmPropagationAsset"); + thermostatAlarms = assetService.saveAsset(thermostatAlarms); + + DeviceId t1Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T1", "T1_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); + DeviceId t2Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T2", "T2_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); + + relationService.saveRelation(thermostatAlarms.getTenantId(), new EntityRelation(thermostatAlarms.getId(), t1Id, "ToAlarmPropagationAsset")); + relationService.saveRelation(thermostatAlarms.getTenantId(), new EntityRelation(thermostatAlarms.getId(), t2Id, "ToAlarmPropagationAsset")); + + attributesService.save(demoTenant.getId(), t1Id, DataConstants.SERVER_SCOPE, + Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.3948)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -122.1503)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 20)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 50)))); + + attributesService.save(demoTenant.getId(), t2Id, DataConstants.SERVER_SCOPE, + Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.493801)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -121.948769)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 25)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 30)))); + installScripts.loadDashboards(demoTenant.getId(), null); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlTimescaleDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/HsqlEntityDatabaseSchemaService.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/install/SqlTimescaleDatabaseSchemaService.java rename to application/src/main/java/org/thingsboard/server/service/install/HsqlEntityDatabaseSchemaService.java index 1db733453b..0b3232903b 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlTimescaleDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/HsqlEntityDatabaseSchemaService.java @@ -17,15 +17,17 @@ package org.thingsboard.server.service.install; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.HsqlDao; import org.thingsboard.server.dao.util.SqlDao; -import org.thingsboard.server.dao.util.TimescaleDBTsDao; @Service -@TimescaleDBTsDao +@HsqlDao +@SqlDao @Profile("install") -public class SqlTimescaleDatabaseSchemaService extends SqlAbstractDatabaseSchemaService - implements TsDatabaseSchemaService { - public SqlTimescaleDatabaseSchemaService() { - super("schema-timescale.sql", "schema-timescale-idx.sql"); +public class HsqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService + implements EntityDatabaseSchemaService { + protected HsqlEntityDatabaseSchemaService() { + super("schema-entities-hsql.sql", "schema-entities-idx.sql"); } -} \ No newline at end of file +} + diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/HsqlTsDatabaseSchemaService.java similarity index 80% rename from application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java rename to application/src/main/java/org/thingsboard/server/service/install/HsqlTsDatabaseSchemaService.java index 1288b0d652..3c877ca636 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/HsqlTsDatabaseSchemaService.java @@ -17,14 +17,16 @@ package org.thingsboard.server.service.install; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.HsqlDao; import org.thingsboard.server.dao.util.SqlTsDao; @Service @SqlTsDao +@HsqlDao @Profile("install") -public class SqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService +public class HsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService { - public SqlTsDatabaseSchemaService() { - super("schema-ts.sql", null); + public HsqlTsDatabaseSchemaService() { + super("schema-ts-hsql.sql", null); } } \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 73e1dbf7fd..5181834b80 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -24,6 +24,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -182,4 +183,30 @@ public class InstallScripts { } + public void loadDemoRuleChains(TenantId tenantId) throws Exception { + Path ruleChainsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, RULE_CHAINS_DIR); + try { + JsonNode ruleChainJson = objectMapper.readTree(ruleChainsDir.resolve("thermostat_alarms.json").toFile()); + RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); + RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); + ruleChain.setTenantId(tenantId); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + ruleChainMetaData.setRuleChainId(ruleChain.getId()); + ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); + + JsonNode rootChainJson = objectMapper.readTree(ruleChainsDir.resolve("root_rule_chain.json").toFile()); + RuleChain rootChain = objectMapper.treeToValue(rootChainJson.get("ruleChain"), RuleChain.class); + RuleChainMetaData rootChainMetaData = objectMapper.treeToValue(rootChainJson.get("metadata"), RuleChainMetaData.class); + + RuleChainId thermostatsRuleChainId = ruleChain.getId(); + rootChainMetaData.getRuleChainConnections().forEach(connection -> connection.setTargetRuleChainId(thermostatsRuleChainId)); + rootChain.setTenantId(tenantId); + rootChain = ruleChainService.saveRuleChain(rootChain); + rootChainMetaData.setRuleChainId(rootChain.getId()); + ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), rootChainMetaData); + } catch (Exception e) { + log.error("Unable to load dashboard from json", e); + throw new RuntimeException("Unable to load dashboard from json", e); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java similarity index 83% rename from application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java rename to application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java index f124b78842..11da8b306a 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java @@ -17,14 +17,16 @@ package org.thingsboard.server.service.install; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.SqlDao; @Service @SqlDao +@PsqlDao @Profile("install") -public class SqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService +public class PsqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements EntityDatabaseSchemaService { - public SqlEntityDatabaseSchemaService() { + public PsqlEntityDatabaseSchemaService() { super("schema-entities.sql", "schema-entities-idx.sql"); } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java new file mode 100644 index 0000000000..bcc3d9bb81 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; + +@Service +@SqlTsDao +@PsqlDao +@Profile("install") +public class PsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService { + + @Value("${sql.postgres.ts_key_value_partitioning:MONTHS}") + private String partitionType; + + public PsqlTsDatabaseSchemaService() { + super("schema-ts-psql.sql", null); + } + + @Override + public void createDatabaseSchema() throws Exception { + super.createDatabaseSchema(); + if (partitionType.equals("INDEFINITE")) { + executeQuery("CREATE TABLE ts_kv_indefinite PARTITION OF ts_kv DEFAULT;"); + } + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java new file mode 100644 index 0000000000..a06ef0fe05 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; + +@Service +@Profile("install") +@Slf4j +@SqlTsDao +@PsqlDao +public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { + + @Value("${sql.postgres.ts_key_value_partitioning:MONTHS}") + private String partitionType; + + private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql"; + private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql"; + private static final String LOAD_DROP_PARTITIONS_FUNCTIONS_SQL = "schema_update_psql_drop_partitions.sql"; + + private static final String TS_KV_OLD = "ts_kv_old;"; + private static final String TS_KV_LATEST_OLD = "ts_kv_latest_old;"; + + private static final String CREATE_PARTITION_TS_KV_TABLE = "create_partition_ts_kv_table()"; + private static final String CREATE_NEW_TS_KV_LATEST_TABLE = "create_new_ts_kv_latest_table()"; + private static final String CREATE_PARTITIONS = "create_partitions(IN partition_type varchar)"; + private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()"; + private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()"; + private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()"; + private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()"; + + private static final String CALL_CREATE_PARTITION_TS_KV_TABLE = CALL_REGEX + CREATE_PARTITION_TS_KV_TABLE; + private static final String CALL_CREATE_NEW_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_LATEST_TABLE; + private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE; + private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY; + private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV; + private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST; + + private static final String DROP_TABLE_TS_KV_OLD = DROP_TABLE + TS_KV_OLD; + private static final String DROP_TABLE_TS_KV_LATEST_OLD = DROP_TABLE + TS_KV_LATEST_OLD; + + private static final String DROP_PROCEDURE_CREATE_PARTITION_TS_KV_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_PARTITION_TS_KV_TABLE; + private static final String DROP_PROCEDURE_CREATE_NEW_TS_KV_LATEST_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_LATEST_TABLE; + private static final String DROP_PROCEDURE_CREATE_PARTITIONS = DROP_PROCEDURE_IF_EXISTS + CREATE_PARTITIONS; + private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE; + private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY; + private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV; + private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST; + private static final String DROP_FUNCTION_GET_PARTITION_DATA = "DROP FUNCTION IF EXISTS get_partitions_data;"; + + @Override + public void upgradeDatabase(String fromVersion) throws Exception { + switch (fromVersion) { + case "2.4.3": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Check the current PostgreSQL version..."); + boolean versionValid = checkVersion(conn); + if (!versionValid) { + throw new RuntimeException("PostgreSQL version should be at least more than 11, please upgrade your PostgreSQL and restart the script!"); + } else { + log.info("PostgreSQL version is valid!"); + if (isOldSchema(conn, 2004003)) { + log.info("Load upgrade functions ..."); + loadSql(conn, LOAD_FUNCTIONS_SQL); + log.info("Updating timeseries schema ..."); + executeQuery(conn, CALL_CREATE_PARTITION_TS_KV_TABLE); + if (!partitionType.equals("INDEFINITE")) { + executeQuery(conn, "call create_partitions('" + partitionType + "')"); + } else { + executeQuery(conn, "CREATE TABLE IF NOT EXISTS ts_kv_indefinite PARTITION OF ts_kv DEFAULT;"); + } + executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); + executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); + executeQuery(conn, CALL_INSERT_INTO_TS_KV); + executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE); + executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); + + executeQuery(conn, DROP_TABLE_TS_KV_OLD); + executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD); + + executeQuery(conn, DROP_PROCEDURE_CREATE_PARTITION_TS_KV_TABLE); + executeQuery(conn, DROP_PROCEDURE_CREATE_PARTITIONS); + executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV); + executeQuery(conn, DROP_PROCEDURE_CREATE_NEW_TS_KV_LATEST_TABLE); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); + executeQuery(conn, DROP_FUNCTION_GET_PARTITION_DATA); + + executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;"); + executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN IF NOT EXISTS json_v json;"); + } else { + executeQuery(conn, "ALTER TABLE ts_kv DROP CONSTRAINT IF EXISTS ts_kv_pkey;"); + executeQuery(conn, "ALTER TABLE ts_kv ADD CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts);"); + } + + log.info("Load TTL functions ..."); + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL); + log.info("Load Drop Partitions functions ..."); + loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL); + + executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005000"); + + log.info("schema timeseries updated!"); + } + } + break; + default: + throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); + } + } + + @Override + protected void loadSql(Connection conn, String fileName) { + Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName); + try { + loadFunctions(schemaUpdateFile, conn); + log.info("Functions successfully loaded!"); + } catch (Exception e) { + log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage()); + } + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java index dda5d2244b..cfc32d31b3 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java @@ -25,6 +25,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.SQLException; @Slf4j public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchemaService { @@ -32,13 +33,13 @@ public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchema private static final String SQL_DIR = "sql"; @Value("${spring.datasource.url}") - private String dbUrl; + protected String dbUrl; @Value("${spring.datasource.username}") - private String dbUserName; + protected String dbUserName; @Value("${spring.datasource.password}") - private String dbPassword; + protected String dbPassword; @Autowired private InstallScripts installScripts; @@ -73,4 +74,14 @@ public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchema } } + protected void executeQuery(String query) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script + log.info("Successfully executed query: {}", query); + Thread.sleep(5000); + } catch (InterruptedException | SQLException e) { + log.info("Failed to execute query: {} due to: {}", query, e.getMessage()); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index b855b09354..87164737c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -30,6 +30,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.SQLSyntaxErrorException; import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; import static org.thingsboard.server.service.install.DatabaseHelper.ASSIGNED_CUSTOMERS; @@ -54,7 +55,7 @@ import static org.thingsboard.server.service.install.DatabaseHelper.TYPE; @Profile("install") @Slf4j @SqlDao -public class SqlDatabaseUpgradeService implements DatabaseUpgradeService { +public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService { private static final String SCHEMA_UPDATE_SQL = "schema_update.sql"; @@ -172,7 +173,8 @@ public class SqlDatabaseUpgradeService implements DatabaseUpgradeService { loadSql(schemaUpdateFile, conn); try { conn.createStatement().execute("ALTER TABLE device ADD COLUMN label varchar(255)"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script - } catch (Exception e) {} + } catch (Exception e) { + } log.info("Schema updated."); } break; @@ -201,7 +203,33 @@ public class SqlDatabaseUpgradeService implements DatabaseUpgradeService { log.info("Updating schema ..."); try { conn.createStatement().execute("ALTER TABLE alarm ADD COLUMN propagate_relation_types varchar"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script - } catch (Exception e) {} + } catch (Exception e) { + } + log.info("Schema updated."); + } + break; + case "2.4.3": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating schema ..."); + try { + conn.createStatement().execute("ALTER TABLE attribute_kv ADD COLUMN json_v json;"); + } catch (Exception e) { + if (e instanceof SQLSyntaxErrorException) { + try { + conn.createStatement().execute("ALTER TABLE attribute_kv ADD COLUMN json_v varchar(10000000);"); + } catch (Exception e1) { + } + } + } + try { + conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); + } catch (Exception e) { + } + try { + long ts = System.currentTimeMillis(); + conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script + } catch (Exception e) { + } log.info("Schema updated."); } break; diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java new file mode 100644 index 0000000000..e2633d6fc4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.TimescaleDBTsDao; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +@Service +@TimescaleDBTsDao +@PsqlDao +@Profile("install") +@Slf4j +public class TimescaleTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService { + + @Value("${sql.timescale.chunk_time_interval:86400000}") + private long chunkTimeInterval; + + public TimescaleTsDatabaseSchemaService() { + super("schema-timescale.sql", null); + } + + @Override + public void createDatabaseSchema() throws Exception { + super.createDatabaseSchema(); + executeQuery("SELECT create_hypertable('ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);"); + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java new file mode 100644 index 0000000000..21bdb500d9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java @@ -0,0 +1,133 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.TimescaleDBTsDao; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; + +@Service +@Profile("install") +@Slf4j +@TimescaleDBTsDao +@PsqlDao +public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { + + @Value("${sql.timescale.chunk_time_interval:86400000}") + private long chunkTimeInterval; + + private static final String LOAD_FUNCTIONS_SQL = "schema_update_timescale_ts.sql"; + private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql"; + + private static final String TENANT_TS_KV_OLD_TABLE = "tenant_ts_kv_old;"; + + private static final String CREATE_TS_KV_LATEST_TABLE = "create_ts_kv_latest_table()"; + private static final String CREATE_NEW_TS_KV_TABLE = "create_new_ts_kv_table()"; + private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()"; + private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()"; + private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()"; + private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()"; + + private static final String CALL_CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_TS_KV_LATEST_TABLE; + private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_TABLE; + private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE; + private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY; + private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV; + private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST; + + private static final String DROP_OLD_TENANT_TS_KV_TABLE = DROP_TABLE + TENANT_TS_KV_OLD_TABLE; + + private static final String DROP_PROCEDURE_CREATE_TS_KV_LATEST_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_LATEST_TABLE; + private static final String DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_TABLE; + private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE; + private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY; + private static final String DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV; + private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST; + + @Autowired + private InstallScripts installScripts; + + @Override + public void upgradeDatabase(String fromVersion) throws Exception { + switch (fromVersion) { + case "2.4.3": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Check the current PostgreSQL version..."); + boolean versionValid = checkVersion(conn); + if (!versionValid) { + throw new RuntimeException("PostgreSQL version should be at least more than 11, please upgrade your PostgreSQL and restart the script!"); + } else { + log.info("PostgreSQL version is valid!"); + if (isOldSchema(conn, 2004003)) { + log.info("Load upgrade functions ..."); + loadSql(conn, LOAD_FUNCTIONS_SQL); + log.info("Updating timescale schema ..."); + executeQuery(conn, CALL_CREATE_TS_KV_LATEST_TABLE); + executeQuery(conn, CALL_CREATE_NEW_TENANT_TS_KV_TABLE); + + executeQuery(conn, "SELECT create_hypertable('ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);"); + + executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); + executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); + executeQuery(conn, CALL_INSERT_INTO_TS_KV); + executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); + + executeQuery(conn, DROP_OLD_TENANT_TS_KV_TABLE); + + executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_LATEST_TABLE); + executeQuery(conn, DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY); + executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); + + executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;"); + executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN IF NOT EXISTS json_v json;"); + } + + log.info("Load TTL functions ..."); + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL); + + executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005000"); + log.info("schema timescale updated!"); + } + } + break; + default: + throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); + } + } + + @Override + protected void loadSql(Connection conn, String fileName) { + Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName); + try { + loadFunctions(schemaUpdateFile, conn); + log.info("Functions successfully loaded!"); + } catch (Exception e) { + log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage()); + } + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java b/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java index 9916c0203a..457d497e57 100644 --- a/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java +++ b/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java @@ -103,7 +103,18 @@ public class DefaultMailService implements MailService { javaMailProperties.put(MAIL_PROP + protocol + ".port", jsonConfig.get("smtpPort").asText()); javaMailProperties.put(MAIL_PROP + protocol + ".timeout", jsonConfig.get("timeout").asText()); javaMailProperties.put(MAIL_PROP + protocol + ".auth", String.valueOf(StringUtils.isNotEmpty(jsonConfig.get("username").asText()))); - javaMailProperties.put(MAIL_PROP + protocol + ".starttls.enable", jsonConfig.has("enableTls") ? jsonConfig.get("enableTls").asText() : "false"); + boolean enableTls = false; + if (jsonConfig.has("enableTls")) { + if (jsonConfig.get("enableTls").isBoolean() && jsonConfig.get("enableTls").booleanValue()) { + enableTls = true; + } else if (jsonConfig.get("enableTls").isTextual()) { + enableTls = "true".equalsIgnoreCase(jsonConfig.get("enableTls").asText()); + } + } + javaMailProperties.put(MAIL_PROP + protocol + ".starttls.enable", enableTls); + if (enableTls && jsonConfig.has("tlsVersion") && StringUtils.isNoneEmpty(jsonConfig.get("tlsVersion").asText())) { + javaMailProperties.put(MAIL_PROP + protocol + ".ssl.protocols", jsonConfig.get("tlsVersion").asText()); + } return javaMailProperties; } @@ -213,7 +224,7 @@ public class DefaultMailService implements MailService { } @Override - public void sendAccountLockoutEmail( String lockoutEmail, String email, Integer maxFailedLoginAttempts) throws ThingsboardException { + public void sendAccountLockoutEmail(String lockoutEmail, String email, Integer maxFailedLoginAttempts) throws ThingsboardException { String subject = messages.getMessage("account.lockout.subject", null, Locale.US); Map model = new HashMap(); @@ -244,7 +255,7 @@ public class DefaultMailService implements MailService { } private static String mergeTemplateIntoString(VelocityEngine velocityEngine, String templateLocation, - String encoding, Map model) throws VelocityException { + String encoding, Map model) throws VelocityException { StringWriter result = new StringWriter(); mergeTemplate(velocityEngine, templateLocation, encoding, model, result); @@ -266,6 +277,7 @@ public class DefaultMailService implements MailService { } else { message = exception.getMessage(); } + log.warn("Unable to send mail: {}", message); return new ThingsboardException(String.format("Unable to send mail: %s", message), ThingsboardErrorCode.GENERAL); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java new file mode 100644 index 0000000000..926b08f38b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -0,0 +1,204 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Service +@Slf4j +public class DefaultTbClusterService implements TbClusterService { + + @Value("${cluster.stats.enabled:false}") + private boolean statsEnabled; + + private final AtomicInteger toCoreMsgs = new AtomicInteger(0); + private final AtomicInteger toCoreNfs = new AtomicInteger(0); + private final AtomicInteger toRuleEngineMsgs = new AtomicInteger(0); + private final AtomicInteger toRuleEngineNfs = new AtomicInteger(0); + private final AtomicInteger toTransportNfs = new AtomicInteger(0); + + private final TbQueueProducerProvider producerProvider; + private final PartitionService partitionService; + private final DataDecodingEncodingService encodingService; + + public DefaultTbClusterService(TbQueueProducerProvider producerProvider, PartitionService partitionService, DataDecodingEncodingService encodingService) { + this.producerProvider = producerProvider; + this.partitionService = partitionService; + this.encodingService = encodingService; + } + + @Override + public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + + @Override + public void pushMsgToCore(TopicPartitionInfo tpi, UUID msgId, ToCoreMsg msg, TbQueueCallback callback) { + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toCoreMsgs.incrementAndGet(); + } + + @Override + public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + log.trace("PUSHING msg: {} to:{}", msg, tpi); + byte[] msgBytes = encodingService.encode(msg); + ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback); + toCoreMsgs.incrementAndGet(); + } + + @Override + public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); + FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() + .setRequestIdMSB(response.getId().getMostSignificantBits()) + .setRequestIdLSB(response.getId().getLeastSignificantBits()) + .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); + response.getResponse().ifPresent(builder::setResponse); + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); + producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); + toCoreNfs.incrementAndGet(); + } + + @Override + public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) { + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toRuleEngineMsgs.incrementAndGet(); + } + + @Override + public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg tbMsg, TbQueueCallback callback) { + if (tenantId.isNullUid()) { + if (entityId.getEntityType().equals(EntityType.TENANT)) { + tenantId = new TenantId(entityId.getId()); + } else { + log.warn("[{}][{}] Received invalid message: {}", tenantId, entityId, tbMsg); + return; + } + } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + log.trace("PUSHING msg: {} to:{}", tbMsg, tpi); + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); + toRuleEngineMsgs.incrementAndGet(); + } + + @Override + public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); + FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() + .setRequestIdMSB(response.getId().getMostSignificantBits()) + .setRequestIdLSB(response.getId().getLeastSignificantBits()) + .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); + response.getResponse().ifPresent(builder::setResponse); + ToRuleEngineNotificationMsg msg = ToRuleEngineNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); + producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); + toRuleEngineNfs.incrementAndGet(); + } + + @Override + public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); + producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback); + toTransportNfs.incrementAndGet(); + } + + @Override + public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { + log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); + broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); + } + + private void broadcast(ComponentLifecycleMsg msg) { + byte[] msgBytes = encodingService.encode(msg); + TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); + Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); + if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)) { + TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); + Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); + for (String serviceId : tbCoreServices) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); + toCoreNfs.incrementAndGet(); + } + // No need to push notifications twice + tbRuleEngineServices.removeAll(tbCoreServices); + } + for (String serviceId : tbRuleEngineServices) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null); + toRuleEngineNfs.incrementAndGet(); + } + } + + @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + int toCoreMsgCnt = toCoreMsgs.getAndSet(0); + int toCoreNfsCnt = toCoreNfs.getAndSet(0); + int toRuleEngineMsgsCnt = toRuleEngineMsgs.getAndSet(0); + int toRuleEngineNfsCnt = toRuleEngineNfs.getAndSet(0); + int toTransportNfsCnt = toTransportNfs.getAndSet(0); + if (toCoreMsgCnt > 0 || toCoreNfsCnt > 0 || toRuleEngineMsgsCnt > 0 || toRuleEngineNfsCnt > 0 || toTransportNfsCnt > 0) { + log.info("To TbCore: [{}] messages [{}] notifications; To TbRuleEngine: [{}] messages [{}] notifications; To Transport: [{}] notifications", + toCoreMsgCnt, toCoreNfsCnt, toRuleEngineMsgsCnt, toRuleEngineNfsCnt, toTransportNfsCnt); + } + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java new file mode 100644 index 0000000000..b7f6bd3dd2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -0,0 +1,297 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import akka.actor.ActorRef; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; +import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; +import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; +import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.subscription.SubscriptionManagerService; +import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; +import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@Slf4j +public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService { + + @Value("${queue.core.poll-interval}") + private long pollDuration; + @Value("${queue.core.pack-processing-timeout}") + private long packProcessingTimeout; + @Value("${queue.core.stats.enabled:false}") + private boolean statsEnabled; + + private final TbQueueConsumer> mainConsumer; + private final DeviceStateService stateService; + private final TbLocalSubscriptionService localSubscriptionService; + private final SubscriptionManagerService subscriptionManagerService; + private final TbCoreDeviceRpcService tbCoreDeviceRpcService; + private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); + + public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, + DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, + SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, + TbCoreDeviceRpcService tbCoreDeviceRpcService) { + super(actorContext, encodingService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); + this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); + this.stateService = stateService; + this.localSubscriptionService = localSubscriptionService; + this.subscriptionManagerService = subscriptionManagerService; + this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; + } + + @PostConstruct + public void init() { + super.init("tb-core-consumer", "tb-core-notifications-consumer"); + } + + @PreDestroy + public void destroy() { + super.destroy(); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(getServiceType())) { + log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); + this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); + } + } + + @Override + protected void launchMainConsumers() { + consumersExecutor.submit(() -> { + while (!stopped) { + try { + List> msgs = mainConsumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + pendingMap.forEach((id, msg) -> { + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + ToCoreMsg toCoreMsg = msg.getValue(); + if (toCoreMsg.hasToSubscriptionMgrMsg()) { + log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); + forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); + } else if (toCoreMsg.hasToDeviceActorMsg()) { + log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); + forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); + } else if (toCoreMsg.hasDeviceStateServiceMsg()) { + log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); + forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); + } else if (toCoreMsg.getToDeviceActorNotificationMsg() != null && !toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); + if (actorMsg.isPresent()) { + TbActorMsg tbActorMsg = actorMsg.get(); + if (tbActorMsg.getMsgType().equals(MsgType.DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG)) { + tbCoreDeviceRpcService.forwardRpcRequestToDeviceActor((ToDeviceRpcRequestActorMsg) tbActorMsg); + } else { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); + } + } + callback.onSuccess(); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + } + mainConsumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + } + log.info("TB Core Consumer stopped."); + }); + } + + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_CORE; + } + + @Override + protected long getNotificationPollDuration() { + return pollDuration; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { + ToCoreNotificationMsg toCoreNotification = msg.getValue(); + if (toCoreNotification.hasToLocalSubscriptionServiceMsg()) { + log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreNotification.getToLocalSubscriptionServiceMsg()); + forwardToLocalSubMgrService(toCoreNotification.getToLocalSubscriptionServiceMsg(), callback); + } else if (toCoreNotification.hasFromDeviceRpcResponse()) { + log.trace("[{}] Forwarding message to RPC service {}", id, toCoreNotification.getFromDeviceRpcResponse()); + forwardToCoreRpcService(toCoreNotification.getFromDeviceRpcResponse(), callback); + } else if (toCoreNotification.getComponentLifecycleMsg() != null && !toCoreNotification.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreNotification.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } + if (statsEnabled) { + stats.log(toCoreNotification); + } + } + + private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { + RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; + FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) + , proto.getResponse(), error); + tbCoreDeviceRpcService.processRpcResponseFromRuleEngine(response); + callback.onSuccess(); + } + + @Scheduled(fixedDelayString = "${queue.core.stats.print-interval-ms}") + public void printStats() { + if (statsEnabled) { + stats.printStats(); + } + } + + private void forwardToLocalSubMgrService(LocalSubscriptionServiceMsgProto msg, TbCallback callback) { + if (msg.hasSubUpdate()) { + localSubscriptionService.onSubscriptionUpdate(msg.getSubUpdate().getSessionId(), TbSubscriptionUtils.fromProto(msg.getSubUpdate()), callback); + } else { + throwNotHandled(msg, callback); + } + } + + private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbCallback callback) { + if (msg.hasAttributeSub()) { + subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getAttributeSub()), callback); + } else if (msg.hasTelemetrySub()) { + subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getTelemetrySub()), callback); + } else if (msg.hasSubClose()) { + TbSubscriptionCloseProto closeProto = msg.getSubClose(); + subscriptionManagerService.cancelSubscription(closeProto.getSessionId(), closeProto.getSubscriptionId(), callback); + } else if (msg.hasTsUpdate()) { + TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); + subscriptionManagerService.onTimeSeriesUpdate( + new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), + TbSubscriptionUtils.toTsKvEntityList(proto.getDataList()), callback); + } else if (msg.hasAttrUpdate()) { + TbAttributeUpdateProto proto = msg.getAttrUpdate(); + subscriptionManagerService.onAttributesUpdate( + new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), + proto.getScope(), TbSubscriptionUtils.toAttributeKvList(proto.getDataList()), callback); + } else { + throwNotHandled(msg, callback); + } + if (statsEnabled) { + stats.log(msg); + } + } + + private void forwardToStateService(DeviceStateServiceMsgProto deviceStateServiceMsg, TbCallback callback) { + if (statsEnabled) { + stats.log(deviceStateServiceMsg); + } + stateService.onQueueMsg(deviceStateServiceMsg, callback); + } + + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbCallback callback) { + if (statsEnabled) { + stats.log(toDeviceActorMsg); + } + actorContext.tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + } + + private void throwNotHandled(Object msg, TbCallback callback) { + log.warn("Message not handled: {}", msg); + callback.onFailure(new RuntimeException("Message not handled!")); + } + + @Override + protected void stopMainConsumers() { + if (mainConsumer != null) { + mainConsumer.unsubscribe(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java new file mode 100644 index 0000000000..182a62d397 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -0,0 +1,278 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import akka.actor.ActorRef; +import com.google.protobuf.ProtocolStringList; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceQueue; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; +import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; +import org.thingsboard.server.service.stats.RuleEngineStatisticsService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Service +@TbRuleEngineComponent +@Slf4j +public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { + + @Value("${queue.rule-engine.poll-interval}") + private long pollDuration; + @Value("${queue.rule-engine.pack-processing-timeout}") + private long packProcessingTimeout; + @Value("${queue.rule-engine.stats.enabled:true}") + private boolean statsEnabled; + + private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; + private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory; + private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final RuleEngineStatisticsService statisticsService; + private final TbRuleEngineDeviceRpcService tbDeviceRpcService; + private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); + private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); + private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); + private ExecutorService submitExecutor; + + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, + TbRuleEngineSubmitStrategyFactory submitStrategyFactory, + TbQueueRuleEngineSettings ruleEngineSettings, + TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, + ActorSystemContext actorContext, DataDecodingEncodingService encodingService, + TbRuleEngineDeviceRpcService tbDeviceRpcService) { + super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); + this.statisticsService = statisticsService; + this.ruleEngineSettings = ruleEngineSettings; + this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; + this.submitStrategyFactory = submitStrategyFactory; + this.processingStrategyFactory = processingStrategyFactory; + this.tbDeviceRpcService = tbDeviceRpcService; + } + + @PostConstruct + public void init() { + super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); + for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) { + consumerConfigurations.putIfAbsent(configuration.getName(), configuration); + consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); + consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName())); + } + submitExecutor = Executors.newSingleThreadExecutor(); + } + + @PreDestroy + public void stop() { + super.destroy(); + if (submitExecutor != null) { + submitExecutor.shutdownNow(); + } + ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config)); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(getServiceType())) { + ServiceQueue serviceQueue = partitionChangeEvent.getServiceQueueKey().getServiceQueue(); + log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), partitionChangeEvent.getPartitions()); + consumers.get(serviceQueue.getQueue()).subscribe(partitionChangeEvent.getPartitions()); + } + } + + @Override + protected void launchMainConsumers() { + consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue))); + } + + @Override + protected void stopMainConsumers() { + consumers.values().forEach(TbQueueConsumer::unsubscribe); + } + + private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats) { + consumersExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = consumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + TbRuleEngineSubmitStrategy submitStrategy = submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy()); + TbRuleEngineProcessingStrategy ackStrategy = processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy()); + + submitStrategy.init(msgs); + + while (!stopped) { + TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(submitStrategy); + submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> { + log.trace("[{}] Creating callback for message: {}", id, msg.getValue()); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); + TbMsgCallback callback = new TbMsgPackCallback(id, tenantId, ctx); + try { + if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { + forwardToRuleEngineActor(tenantId, toRuleEngineMsg, callback); + } else { + callback.onSuccess(); + } + } catch (Exception e) { + callback.onFailure(new RuleEngineException(e.getMessage())); + } + })); + + boolean timeout = false; + if (!ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { + timeout = true; + } + + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(timeout, ctx); + TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); + if (statsEnabled) { + stats.log(result, decision.isCommit()); + } + if (decision.isCommit()) { + submitStrategy.stop(); + break; + } else { + submitStrategy.update(decision.getReprocessMap()); + } + } + consumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to process messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + } + log.info("TB Rule Engine Consumer stopped."); + }); + } + + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_RULE_ENGINE; + } + + @Override + protected long getNotificationPollDuration() { + return pollDuration; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception { + ToRuleEngineNotificationMsg nfMsg = msg.getValue(); + if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(nfMsg.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } else if (nfMsg.hasFromDeviceRpcResponse()) { + TransportProtos.FromDeviceRPCResponseProto proto = nfMsg.getFromDeviceRpcResponse(); + RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; + FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) + , proto.getResponse(), error); + tbDeviceRpcService.processRpcResponseFromDevice(response); + callback.onSuccess(); + } else { + log.trace("Received notification with missing handler"); + callback.onSuccess(); + } + } + + private void forwardToRuleEngineActor(TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + TbMsg tbMsg = TbMsg.fromBytes(toRuleEngineMsg.getTbMsg().toByteArray(), callback); + QueueToRuleEngineMsg msg; + ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); + Set relationTypes = null; + if (relationTypesList != null) { + if (relationTypesList.size() == 1) { + relationTypes = Collections.singleton(relationTypesList.get(0)); + } else { + relationTypes = new HashSet<>(relationTypesList); + } + } + msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); + actorContext.tell(msg, ActorRef.noSender()); + } + + @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") + public void printStats() { + if (statsEnabled) { + long ts = System.currentTimeMillis(); + consumerStats.forEach((queue, stats) -> { + stats.printStats(); + statisticsService.reportQueueStats(ts, stats); + }); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java new file mode 100644 index 0000000000..aae3cef0ac --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'") +public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService { + + private final TenantService tenantService; + + public DefaultTenantRoutingInfoService(TenantService tenantService) { + this.tenantService = tenantService; + } + + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + Tenant tenant = tenantService.findTenantById(tenantId); + if (tenant != null) { + return new TenantRoutingInfo(tenantId, tenant.isIsolatedTbCore(), tenant.isIsolatedTbRuleEngine()); + } else { + throw new RuntimeException("Tenant not found!"); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java new file mode 100644 index 0000000000..cc722720d5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; + +import java.util.UUID; + +public interface TbClusterService { + + void pushMsgToCore(TopicPartitionInfo tpi, UUID msgKey, ToCoreMsg msg, TbQueueCallback callback); + + void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback); + + void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback); + + void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); + + void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, TransportProtos.ToRuleEngineMsg msg, TbQueueCallback callback); + + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg msg, TbQueueCallback callback); + + void pushNotificationToRuleEngine(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); + + void pushNotificationToTransport(String targetServiceId, ToTransportMsg response, TbQueueCallback callback); + + void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java new file mode 100644 index 0000000000..2dcdb0e0c2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; + +public interface TbCoreConsumerService extends ApplicationListener { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java similarity index 63% rename from application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index dea6e6d33c..c362c5e889 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; @@ -21,31 +21,26 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -public class RuleEngineStats { +public class TbCoreConsumerStats { private final AtomicInteger totalCounter = new AtomicInteger(0); private final AtomicInteger sessionEventCounter = new AtomicInteger(0); - private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); - private final AtomicInteger postAttributesCounter = new AtomicInteger(0); private final AtomicInteger getAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); - private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); + private final AtomicInteger deviceStateCounter = new AtomicInteger(0); + private final AtomicInteger subscriptionMsgCounter = new AtomicInteger(0); + private final AtomicInteger toCoreNotificationsCounter = new AtomicInteger(0); + public void log(TransportProtos.TransportToDeviceActorMsg msg) { totalCounter.incrementAndGet(); if (msg.hasSessionEvent()) { sessionEventCounter.incrementAndGet(); } - if (msg.hasPostTelemetry()) { - postTelemetryCounter.incrementAndGet(); - } - if (msg.hasPostAttributes()) { - postAttributesCounter.incrementAndGet(); - } if (msg.hasGetAttributes()) { getAttributesCounter.incrementAndGet(); } @@ -58,9 +53,6 @@ public class RuleEngineStats { if (msg.hasToDeviceRPCCallResponse()) { toDeviceRPCCallResponseCounter.incrementAndGet(); } - if (msg.hasToServerRPCCallRequest()) { - toServerRPCCallRequestCounter.incrementAndGet(); - } if (msg.hasSubscriptionInfo()) { subscriptionInfoCounter.incrementAndGet(); } @@ -69,15 +61,32 @@ public class RuleEngineStats { } } + public void log(TransportProtos.DeviceStateServiceMsgProto msg) { + totalCounter.incrementAndGet(); + deviceStateCounter.incrementAndGet(); + } + + public void log(TransportProtos.SubscriptionMgrMsgProto msg) { + totalCounter.incrementAndGet(); + subscriptionMsgCounter.incrementAndGet(); + } + + public void log(TransportProtos.ToCoreNotificationMsg msg) { + totalCounter.incrementAndGet(); + toCoreNotificationsCounter.incrementAndGet(); + } + public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { - log.info("Transport total [{}] sessionEvents [{}] telemetry [{}] attributes [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] " + - "toServerRpc [{}] subInfo [{}] claimDevice [{}] ", - total, sessionEventCounter.getAndSet(0), postTelemetryCounter.getAndSet(0), - postAttributesCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), + log.info("Total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + + " deviceState [{}] subMgr [{}] coreNfs [{}]", + total, sessionEventCounter.getAndSet(0), + getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), - toServerRPCCallRequestCounter.getAndSet(0), subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); + subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0) + , deviceStateCounter.getAndSet(0), subscriptionMsgCounter.getAndSet(0), toCoreNotificationsCounter.getAndSet(0)); } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java new file mode 100644 index 0000000000..f093cc885a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; + +import java.util.UUID; + +@Slf4j +public class TbMsgPackCallback implements TbMsgCallback { + private final UUID id; + private final TenantId tenantId; + private final TbMsgPackProcessingContext ctx; + + public TbMsgPackCallback(UUID id, TenantId tenantId, TbMsgPackProcessingContext ctx) { + this.id = id; + this.tenantId = tenantId; + this.ctx = ctx; + } + + @Override + public void onSuccess() { + log.trace("[{}] ON SUCCESS", id); + ctx.onSuccess(id); + } + + @Override + public void onFailure(RuleEngineException e) { + log.trace("[{}] ON FAILURE", id, e); + ctx.onFailure(tenantId, id, e); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java new file mode 100644 index 0000000000..7e78ac6f5f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class TbMsgPackProcessingContext { + + private final TbRuleEngineSubmitStrategy submitStrategy; + + private final AtomicInteger pendingCount; + private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + @Getter + private final ConcurrentMap> pendingMap; + @Getter + private final ConcurrentMap> successMap = new ConcurrentHashMap<>(); + @Getter + private final ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + @Getter + private final ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); + + public TbMsgPackProcessingContext(TbRuleEngineSubmitStrategy submitStrategy) { + this.submitStrategy = submitStrategy; + this.pendingMap = submitStrategy.getPendingMap(); + this.pendingCount = new AtomicInteger(pendingMap.size()); + } + + public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { + return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); + } + + public void onSuccess(UUID id) { + TbProtoQueueMsg msg; + boolean empty = false; + msg = pendingMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + successMap.put(id, msg); + submitStrategy.onSuccess(id); + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } + + public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) { + TbProtoQueueMsg msg; + boolean empty = false; + msg = pendingMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + failedMap.put(id, msg); + exceptionsMap.putIfAbsent(tenantId, e); + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java similarity index 50% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index 80674f9bce..5a5c172ee6 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -13,35 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.service.queue; -import lombok.Data; -import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TbCallback; -import java.io.Serializable; +import java.util.UUID; -/** - * @author Andrew Shvayka - */ -@Data -@EqualsAndHashCode -public class ServerAddress implements Comparable, Serializable { +@Slf4j +public class TbPackCallback implements TbCallback { + private final TbPackProcessingContext ctx; + private final UUID id; - private final String host; - private final int port; - private final ServerType serverType; + public TbPackCallback(UUID id, TbPackProcessingContext ctx) { + this.id = id; + this.ctx = ctx; + } @Override - public int compareTo(ServerAddress o) { - int result = this.host.compareTo(o.host); - if (result == 0) { - result = this.port - o.port; - } - return result; + public void onSuccess() { + log.trace("[{}] ON SUCCESS", id); + ctx.onSuccess(id); } @Override - public String toString() { - return '[' + host + ':' + port + ']'; + public void onFailure(Throwable t) { + log.trace("[{}] ON FAILURE", id, t); + ctx.onFailure(id, t); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java new file mode 100644 index 0000000000..e9f1224625 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class TbPackProcessingContext { + + private final AtomicInteger pendingCount; + private final CountDownLatch processingTimeoutLatch; + private final ConcurrentMap ackMap; + private final ConcurrentMap failedMap; + + public TbPackProcessingContext(CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap failedMap) { + this.processingTimeoutLatch = processingTimeoutLatch; + this.pendingCount = new AtomicInteger(ackMap.size()); + this.ackMap = ackMap; + this.failedMap = failedMap; + } + + public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { + return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); + } + + public void onSuccess(UUID id) { + boolean empty = false; + T msg = ackMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + } + if (empty) { + processingTimeoutLatch.countDown(); + } else { + if (log.isTraceEnabled()) { + log.trace("Items left: {}", ackMap.size()); + for (T t : ackMap.values()) { + log.trace("left item: {}", t); + } + } + } + } + + public void onFailure(UUID id, Throwable t) { + boolean empty = false; + T msg = ackMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + failedMap.put(id, msg); + if (log.isTraceEnabled()) { + log.trace("Items left: {}", ackMap.size()); + for (T v : ackMap.values()) { + log.trace("left item: {}", v); + } + } + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } + + public ConcurrentMap getAckMap() { + return ackMap; + } + + public ConcurrentMap getFailedMap() { + return failedMap; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java new file mode 100644 index 0000000000..19966b4566 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; + +public interface TbRuleEngineConsumerService extends ApplicationListener { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java new file mode 100644 index 0000000000..40017d2b40 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -0,0 +1,132 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +@Data +public class TbRuleEngineConsumerStats { + + public static final String TOTAL_MSGS = "totalMsgs"; + public static final String SUCCESSFUL_MSGS = "successfulMsgs"; + public static final String TMP_TIMEOUT = "tmpTimeout"; + public static final String TMP_FAILED = "tmpFailed"; + public static final String TIMEOUT_MSGS = "timeoutMsgs"; + public static final String FAILED_MSGS = "failedMsgs"; + public static final String SUCCESSFUL_ITERATIONS = "successfulIterations"; + public static final String FAILED_ITERATIONS = "failedIterations"; + + private final AtomicInteger totalMsgCounter = new AtomicInteger(0); + private final AtomicInteger successMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger failedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger successIterationsCounter = new AtomicInteger(0); + private final AtomicInteger failedIterationsCounter = new AtomicInteger(0); + + private final Map counters = new HashMap<>(); + private final ConcurrentMap tenantStats = new ConcurrentHashMap<>(); + private final ConcurrentMap tenantExceptions = new ConcurrentHashMap<>(); + + private final String queueName; + + public TbRuleEngineConsumerStats(String queueName) { + this.queueName = queueName; + counters.put(TOTAL_MSGS, totalMsgCounter); + counters.put(SUCCESSFUL_MSGS, successMsgCounter); + counters.put(TIMEOUT_MSGS, timeoutMsgCounter); + counters.put(FAILED_MSGS, failedMsgCounter); + + counters.put(TMP_TIMEOUT, tmpTimeoutMsgCounter); + counters.put(TMP_FAILED, tmpFailedMsgCounter); + counters.put(SUCCESSFUL_ITERATIONS, successIterationsCounter); + counters.put(FAILED_ITERATIONS, failedIterationsCounter); + } + + public void log(TbRuleEngineProcessingResult msg, boolean finalIterationForPack) { + int success = msg.getSuccessMap().size(); + int pending = msg.getPendingMap().size(); + int failed = msg.getFailedMap().size(); + totalMsgCounter.addAndGet(success + pending + failed); + successMsgCounter.addAndGet(success); + msg.getSuccessMap().values().forEach(m -> getTenantStats(m).logSuccess()); + if (finalIterationForPack) { + if (pending > 0 || failed > 0) { + timeoutMsgCounter.addAndGet(pending); + failedMsgCounter.addAndGet(failed); + if (pending > 0) { + msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTimeout()); + } + if (failed > 0) { + msg.getFailedMap().values().forEach(m -> getTenantStats(m).logFailed()); + } + failedIterationsCounter.incrementAndGet(); + } else { + successIterationsCounter.incrementAndGet(); + } + } else { + failedIterationsCounter.incrementAndGet(); + tmpTimeoutMsgCounter.addAndGet(pending); + tmpFailedMsgCounter.addAndGet(failed); + if (pending > 0) { + msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTmpTimeout()); + } + if (failed > 0) { + msg.getFailedMap().values().forEach(m -> getTenantStats(m).logTmpFailed()); + } + } + msg.getExceptionsMap().forEach(tenantExceptions::putIfAbsent); + } + + private TbTenantRuleEngineStats getTenantStats(TbProtoQueueMsg m) { + ToRuleEngineMsg reMsg = m.getValue(); + return tenantStats.computeIfAbsent(new UUID(reMsg.getTenantIdMSB(), reMsg.getTenantIdLSB()), TbTenantRuleEngineStats::new); + } + + public void printStats() { + int total = totalMsgCounter.get(); + if (total > 0) { + StringBuilder stats = new StringBuilder(); + counters.forEach((label, value) -> { + stats.append(label).append(" = [").append(value.get()).append("] "); + }); + log.info("[{}] Stats: {}", queueName, stats); + } + } + + public void reset() { + counters.values().forEach(counter -> counter.set(0)); + tenantStats.clear(); + tenantExceptions.clear(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java new file mode 100644 index 0000000000..fb36f5064b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +@Data +public class TbTenantRuleEngineStats { + + private final UUID tenantId; + + private final AtomicInteger totalMsgCounter = new AtomicInteger(0); + private final AtomicInteger successMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger failedMsgCounter = new AtomicInteger(0); + + private final Map counters = new HashMap<>(); + + public TbTenantRuleEngineStats(UUID tenantId) { + this.tenantId = tenantId; + counters.put(TbRuleEngineConsumerStats.TOTAL_MSGS, totalMsgCounter); + counters.put(TbRuleEngineConsumerStats.SUCCESSFUL_MSGS, successMsgCounter); + counters.put(TbRuleEngineConsumerStats.TIMEOUT_MSGS, timeoutMsgCounter); + counters.put(TbRuleEngineConsumerStats.FAILED_MSGS, failedMsgCounter); + + counters.put(TbRuleEngineConsumerStats.TMP_TIMEOUT, tmpTimeoutMsgCounter); + counters.put(TbRuleEngineConsumerStats.TMP_FAILED, tmpFailedMsgCounter); + } + + public void logSuccess() { + totalMsgCounter.incrementAndGet(); + successMsgCounter.incrementAndGet(); + } + + public void logFailed() { + totalMsgCounter.incrementAndGet(); + failedMsgCounter.incrementAndGet(); + } + + public void logTimeout() { + totalMsgCounter.incrementAndGet(); + timeoutMsgCounter.incrementAndGet(); + } + + public void logTmpFailed() { + totalMsgCounter.incrementAndGet(); + tmpFailedMsgCounter.incrementAndGet(); + } + + public void logTmpTimeout() { + totalMsgCounter.incrementAndGet(); + tmpTimeoutMsgCounter.incrementAndGet(); + } + + public void printStats() { + int total = totalMsgCounter.get(); + if (total > 0) { + StringBuilder stats = new StringBuilder(); + counters.forEach((label, value) -> { + stats.append(label).append(" = [").append(value.get()).append("]"); + }); + log.info("[{}] Stats: {}", tenantId, stats); + } + } + + public void reset() { + counters.values().forEach(counter -> counter.set(0)); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java new file mode 100644 index 0000000000..4007c9e17d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -0,0 +1,148 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.EventListener; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.TbPackCallback; +import org.thingsboard.server.service.queue.TbPackProcessingContext; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public abstract class AbstractConsumerService implements ApplicationListener { + + protected volatile ExecutorService consumersExecutor; + protected volatile ExecutorService notificationsConsumerExecutor; + protected volatile boolean stopped = false; + + protected final ActorSystemContext actorContext; + protected final DataDecodingEncodingService encodingService; + + protected final TbQueueConsumer> nfConsumer; + + public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbQueueConsumer> nfConsumer) { + this.actorContext = actorContext; + this.encodingService = encodingService; + this.nfConsumer = nfConsumer; + } + + public void init(String mainConsumerThreadName, String nfConsumerThreadName) { + this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName(mainConsumerThreadName)); + this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(nfConsumerThreadName)); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); + this.nfConsumer.subscribe(); + launchNotificationsConsumer(); + launchMainConsumers(); + } + + protected abstract ServiceType getServiceType(); + + protected abstract void launchMainConsumers(); + + protected abstract void stopMainConsumers(); + + protected abstract long getNotificationPollDuration(); + + protected abstract long getNotificationPackProcessingTimeout(); + + protected void launchNotificationsConsumer() { + notificationsConsumerExecutor.submit(() -> { + while (!stopped) { + try { + List> msgs = nfConsumer.poll(getNotificationPollDuration()); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + pendingMap.forEach((id, msg) -> { + log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + handleNotification(id, msg, callback); + } catch (Throwable e) { + log.warn("[{}] Failed to process notification: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); + } + nfConsumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain notifications from queue.", e); + try { + Thread.sleep(getNotificationPollDuration()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new notifications", e2); + } + } + } + } + log.info("TB Notifications Consumer stopped."); + }); + } + + protected abstract void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception; + + @PreDestroy + public void destroy() { + stopped = true; + + stopMainConsumers(); + + if (nfConsumer != null) { + nfConsumer.unsubscribe(); + } + + if (consumersExecutor != null) { + consumersExecutor.shutdownNow(); + } + if (notificationsConsumerExecutor != null) { + notificationsConsumerExecutor.shutdownNow(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..bef733ec22 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngineSubmitStrategy { + + protected final String queueName; + protected List orderedMsgList; + private volatile boolean stopped; + + public AbstractTbRuleEngineSubmitStrategy(String queueName) { + this.queueName = queueName; + } + + protected abstract void doOnSuccess(UUID id); + + @Override + public void init(List> msgs) { + orderedMsgList = msgs.stream().map(msg -> new IdMsgPair(UUID.randomUUID(), msg)).collect(Collectors.toList()); + } + + @Override + public ConcurrentMap> getPendingMap() { + return orderedMsgList.stream().collect(Collectors.toConcurrentMap(pair -> pair.uuid, pair -> pair.msg)); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + List newOrderedMsgList = new ArrayList<>(reprocessMap.size()); + for (IdMsgPair pair : orderedMsgList) { + if (reprocessMap.containsKey(pair.uuid)) { + newOrderedMsgList.add(pair); + } + } + orderedMsgList = newOrderedMsgList; + } + + @Override + public void onSuccess(UUID id) { + if (!stopped) { + doOnSuccess(id); + } + } + + @Override + public void stop() { + stopped = true; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..b9741d2433 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java @@ -0,0 +1,85 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +@Slf4j +public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private final int batchSize; + private final AtomicInteger packIdx = new AtomicInteger(0); + private final Map> pendingPack = new LinkedHashMap<>(); + private volatile BiConsumer> msgConsumer; + + public BatchTbRuleEngineSubmitStrategy(String queueName, int batchSize) { + super(queueName); + this.batchSize = batchSize; + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + submitNext(); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + packIdx.set(0); + } + + @Override + protected void doOnSuccess(UUID id) { + boolean endOfPendingPack; + synchronized (pendingPack) { + TbProtoQueueMsg msg = pendingPack.remove(id); + endOfPendingPack = msg != null && pendingPack.isEmpty(); + } + if (endOfPendingPack) { + packIdx.incrementAndGet(); + submitNext(); + } + } + + private void submitNext() { + int listSize = orderedMsgList.size(); + int startIdx = Math.min(packIdx.get() * batchSize, listSize); + int endIdx = Math.min(startIdx + batchSize, listSize); + synchronized (pendingPack) { + pendingPack.clear(); + for (int i = startIdx; i < endIdx; i++) { + IdMsgPair pair = orderedMsgList.get(i); + pendingPack.put(pair.uuid, pair.msg); + } + } + int submitSize = pendingPack.size(); + if (log.isDebugEnabled() && submitSize > 0) { + log.debug("[{}] submitting [{}] messages to rule engine", queueName, submitSize); + } + pendingPack.forEach(msgConsumer); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..3420933d3a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.function.BiConsumer; + +@Slf4j +public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + public BurstTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + if (log.isDebugEnabled()) { + log.debug("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); + } + orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg)); + } + + @Override + protected void doOnSuccess(UUID id) { + + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java similarity index 64% rename from application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java rename to application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java index 9f08463f91..2b2c203ec5 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.service.queue.processing; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.io.IOException; +import java.util.UUID; -/** - * Created by ashvayka on 05.10.18. - */ -public class ToRuleEngineMsgDecoder implements TbKafkaDecoder { - @Override - public ToRuleEngineMsg decode(byte[] data) throws IOException { - return ToRuleEngineMsg.parseFrom(data); +public class IdMsgPair { + final UUID uuid; + final TbProtoQueueMsg msg; + + public IdMsgPair(UUID uuid, TbProtoQueueMsg msg) { + this.uuid = uuid; + this.msg = msg; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..473810b86c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,100 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; + +@Slf4j +public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private volatile BiConsumer> msgConsumer; + private volatile ConcurrentMap msgToEntityIdMap = new ConcurrentHashMap<>(); + private volatile ConcurrentMap> entityIdToListMap = new ConcurrentHashMap<>(); + + public SequentialByEntityIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void init(List> msgs) { + super.init(msgs); + initMaps(); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + entityIdToListMap.forEach((entityId, queue) -> { + IdMsgPair msg = queue.peek(); + if (msg != null) { + msgConsumer.accept(msg.uuid, msg.msg); + } + }); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + initMaps(); + } + + @Override + protected void doOnSuccess(UUID id) { + EntityId entityId = msgToEntityIdMap.get(id); + if (entityId != null) { + Queue queue = entityIdToListMap.get(entityId); + if (queue != null) { + IdMsgPair next = null; + synchronized (queue) { + IdMsgPair expected = queue.peek(); + if (expected != null && expected.uuid.equals(id)) { + queue.poll(); + next = queue.peek(); + } + } + if (next != null) { + msgConsumer.accept(next.uuid, next.msg); + } + } + } + } + + private void initMaps() { + msgToEntityIdMap.clear(); + entityIdToListMap.clear(); + for (IdMsgPair pair : orderedMsgList) { + EntityId entityId = getEntityId(pair.msg.getValue()); + if (entityId != null) { + msgToEntityIdMap.put(pair.uuid, entityId); + entityIdToListMap.computeIfAbsent(entityId, id -> new LinkedList<>()).add(pair); + } + } + } + + protected abstract EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..cd8a97e82c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.UUID; + +@Slf4j +public class SequentialByOriginatorIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { + + public SequentialByOriginatorIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { + try { + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg()); + return EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + } catch (InvalidProtocolBufferException e) { + log.warn("[{}] Failed to parse TbMsg: {}", queueName, msg); + return null; + } + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java similarity index 55% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java rename to application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java index 3326c33aa3..37e9419edd 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java @@ -13,28 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.service.queue.processing; -import lombok.Data; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos; -@Data -public class SendToClusterMsg implements TbActorMsg { +import java.util.UUID; - private TbActorMsg msg; - private EntityId entityId; +public class SequentialByTenantIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { - public SendToClusterMsg(EntityId entityId, TbActorMsg msg) { - this.entityId = entityId; - this.msg = msg; + public SequentialByTenantIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); } - @Override - public MsgType getMsgType() { - return MsgType.SEND_TO_CLUSTER_MSG; + protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { + return new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..125a1d8ef8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +@Slf4j +public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private final AtomicInteger msgIdx = new AtomicInteger(0); + private volatile BiConsumer> msgConsumer; + private volatile UUID expectedMsgId; + + public SequentialTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + msgIdx.set(0); + submitNext(); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + } + + @Override + protected void doOnSuccess(UUID id) { + if (expectedMsgId.equals(id)) { + msgIdx.incrementAndGet(); + submitNext(); + } + } + + private void submitNext() { + int listSize = orderedMsgList.size(); + int idx = msgIdx.get(); + if (idx < listSize) { + IdMsgPair pair = orderedMsgList.get(idx); + expectedMsgId = pair.uuid; + if (log.isDebugEnabled()) { + log.debug("[{}] submitting [{}] message to rule engine", queueName, pair.msg); + } + msgConsumer.accept(pair.uuid, pair.msg); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java new file mode 100644 index 0000000000..4a4829f0c2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.Data; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; + +@Data +public class TbRuleEngineProcessingDecision { + + private final boolean commit; + private final ConcurrentMap> reprocessMap; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java new file mode 100644 index 0000000000..e818ffb9d7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; + +public class TbRuleEngineProcessingResult { + + @Getter + private final boolean success; + @Getter + private final boolean timeout; + @Getter + private final TbMsgPackProcessingContext ctx; + + public TbRuleEngineProcessingResult(boolean timeout, TbMsgPackProcessingContext ctx) { + this.timeout = timeout; + this.ctx = ctx; + this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty(); + } + + public ConcurrentMap> getPendingMap() { + return ctx.getPendingMap(); + } + + public ConcurrentMap> getSuccessMap() { + return ctx.getSuccessMap(); + } + + public ConcurrentMap> getFailedMap() { + return ctx.getFailedMap(); + } + + public ConcurrentMap getExceptionsMap() { + return ctx.getExceptionsMap(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java new file mode 100644 index 0000000000..cfba85f5dc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +public interface TbRuleEngineProcessingStrategy { + + TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java new file mode 100644 index 0000000000..80b0523a81 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -0,0 +1,140 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +@Component +@Slf4j +public class TbRuleEngineProcessingStrategyFactory { + + public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) { + switch (configuration.getType()) { + case "SKIP_ALL_FAILURES": + return new SkipStrategy(name); + case "RETRY_ALL": + return new RetryStrategy(name, true, true, true, configuration); + case "RETRY_FAILED": + return new RetryStrategy(name, false, true, false, configuration); + case "RETRY_TIMED_OUT": + return new RetryStrategy(name, false, false, true, configuration); + case "RETRY_FAILED_AND_TIMED_OUT": + return new RetryStrategy(name, false, true, true, configuration); + default: + throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); + } + } + + private static class RetryStrategy implements TbRuleEngineProcessingStrategy { + private final String queueName; + private final boolean retrySuccessful; + private final boolean retryFailed; + private final boolean retryTimeout; + private final int maxRetries; + private final double maxAllowedFailurePercentage; + private final long pauseBetweenRetries; + + private int initialTotalCount; + private int retryCount; + + public RetryStrategy(String queueName, boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, TbRuleEngineQueueAckStrategyConfiguration configuration) { + this.queueName = queueName; + this.retrySuccessful = retrySuccessful; + this.retryFailed = retryFailed; + this.retryTimeout = retryTimeout; + this.maxRetries = configuration.getRetries(); + this.maxAllowedFailurePercentage = configuration.getFailurePercentage(); + this.pauseBetweenRetries = configuration.getPauseBetweenRetries(); + } + + @Override + public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { + if (result.isSuccess()) { + return new TbRuleEngineProcessingDecision(true, null); + } else { + if (retryCount == 0) { + initialTotalCount = result.getPendingMap().size() + result.getFailedMap().size() + result.getSuccessMap().size(); + } + retryCount++; + double failedCount = result.getFailedMap().size() + result.getPendingMap().size(); + if (maxRetries > 0 && retryCount > maxRetries) { + log.debug("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); + return new TbRuleEngineProcessingDecision(true, null); + } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { + log.debug("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); + return new TbRuleEngineProcessingDecision(true, null); + } else { + ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); + if (retryFailed) { + result.getFailedMap().forEach(toReprocess::put); + } + if (retryTimeout) { + result.getPendingMap().forEach(toReprocess::put); + } + if (retrySuccessful) { + result.getSuccessMap().forEach(toReprocess::put); + } + log.debug("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); + if (log.isTraceEnabled()) { + toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + if (pauseBetweenRetries > 0) { + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(pauseBetweenRetries)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return new TbRuleEngineProcessingDecision(false, toReprocess); + } + } + } + } + + private static class SkipStrategy implements TbRuleEngineProcessingStrategy { + + private final String queueName; + + public SkipStrategy(String name) { + this.queueName = name; + } + + @Override + public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { + if (!result.isSuccess()) { + log.debug("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + } + if (log.isTraceEnabled()) { + result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + if (log.isTraceEnabled()) { + result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + return new TbRuleEngineProcessingDecision(true, null); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..7b22da97db --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; + +public interface TbRuleEngineSubmitStrategy { + + void init(List> msgs); + + ConcurrentMap> getPendingMap(); + + void submitAttempt(BiConsumer> msgConsumer); + + void update(ConcurrentMap> reprocessMap); + + void onSuccess(UUID id); + + void stop(); +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java new file mode 100644 index 0000000000..90eb8561a0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueSubmitStrategyConfiguration; + +@Component +@Slf4j +public class TbRuleEngineSubmitStrategyFactory { + + public TbRuleEngineSubmitStrategy newInstance(String name, TbRuleEngineQueueSubmitStrategyConfiguration configuration) { + switch (configuration.getType()) { + case "BURST": + return new BurstTbRuleEngineSubmitStrategy(name); + case "BATCH": + return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize()); + case "SEQUENTIAL_BY_ORIGINATOR": + return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name); + case "SEQUENTIAL_BY_TENANT": + return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name); + case "SEQUENTIAL": + return new SequentialTbRuleEngineSubmitStrategy(name); + default: + throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java deleted file mode 100644 index 5d303b54ab..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java +++ /dev/null @@ -1,222 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; -import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 27.03.18. - */ -@Service -@Slf4j -public class DefaultDeviceRpcService implements DeviceRpcService { - - private static final ObjectMapper json = new ObjectMapper(); - - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService rpcService; - - @Autowired - private DeviceService deviceService; - - @Autowired - @Lazy - private ActorService actorService; - - private ScheduledExecutorService rpcCallBackExecutor; - - private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); - private final ConcurrentMap> localToDeviceRpcRequests = new ConcurrentHashMap<>(); - - @PostConstruct - public void initExecutor() { - rpcCallBackExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rpc-callback")); - } - - @PreDestroy - public void shutdownExecutor() { - if (rpcCallBackExecutor != null) { - rpcCallBackExecutor.shutdownNow(); - } - } - - @Override - public void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToRuleEngineRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToRuleEngine(request); - scheduleTimeout(request, requestId, localToRuleEngineRpcRequests); - } - - @Override - public void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), requestOriginAddress); - if (routingService.getCurrentServer().equals(requestOriginAddress)) { - UUID requestId = response.getId(); - Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } else { - ClusterAPIProtos.FromDeviceRPCResponseProto.Builder builder = ClusterAPIProtos.FromDeviceRPCResponseProto.newBuilder(); - builder.setRequestIdMSB(response.getId().getMostSignificantBits()); - builder.setRequestIdLSB(response.getId().getLeastSignificantBits()); - response.getResponse().ifPresent(builder::setResponse); - if (response.getError().isPresent()) { - builder.setError(response.getError().get().ordinal()); - } else { - builder.setError(-1); - } - rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); - } - } - - @Override - public void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToDeviceRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToDevice(request); - scheduleTimeout(request, requestId, localToDeviceRpcRequests); - } - - @Override - public void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); - UUID requestId = response.getId(); - Consumer consumer = localToDeviceRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } - - @Override - public void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.FromDeviceRPCResponseProto proto; - try { - proto = ClusterAPIProtos.FromDeviceRPCResponseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; - FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), proto.getResponse(), error); - processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); - } - - @Override - public void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body) { - ToServerRpcResponseActorMsg rpcMsg = new ToServerRpcResponseActorMsg(tenantId, deviceId, new ToServerRpcResponseMsg(requestId, body)); - forward(deviceId, rpcMsg); - } - - private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { - ObjectNode entityNode = json.createObjectNode(); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("requestUUID", msg.getId().toString()); - metaData.putValue("originHost", routingService.getCurrentServer().getHost()); - metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); - metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); - metaData.putValue("oneway", Boolean.toString(msg.isOneway())); - - Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); - if (device != null) { - metaData.putValue("deviceName", device.getName()); - metaData.putValue("deviceType", device.getType()); - } - - entityNode.put("method", msg.getBody().getMethod()); - entityNode.put("params", msg.getBody().getParams()); - - try { - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON - , json.writeValueAsString(entityNode) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new ServiceToRuleEngineMsg(msg.getTenantId(), tbMsg))); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { - ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); - log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); - forward(msg.getDeviceId(), rpcMsg); - } - - private void forward(DeviceId deviceId, T msg) { - actorService.onMsg(new SendToClusterMsg(deviceId, msg)); - } - - private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId, ConcurrentMap> requestsMap) { - long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); - rpcCallBackExecutor.schedule(() -> { - log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); - Consumer consumer = requestsMap.remove(requestId); - if (consumer != null) { - consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); - } - }, timeout, TimeUnit.MILLISECONDS); - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java new file mode 100644 index 0000000000..f6e5eb8b43 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -0,0 +1,199 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import akka.actor.ActorRef; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 27.03.18. + */ +@Service +@Slf4j +@TbCoreComponent +public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { + + private static final ObjectMapper json = new ObjectMapper(); + + private final DeviceService deviceService; + private final TbClusterService clusterService; + private final TbServiceInfoProvider serviceInfoProvider; + private final ActorSystemContext actorContext; + + private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); + private final ConcurrentMap localToDeviceRpcRequests = new ConcurrentHashMap<>(); + + private Optional tbRuleEngineRpcService; + private ScheduledExecutorService scheduler; + private String serviceId; + + public DefaultTbCoreDeviceRpcService(DeviceService deviceService, TbClusterService clusterService, TbServiceInfoProvider serviceInfoProvider, + ActorSystemContext actorContext) { + this.deviceService = deviceService; + this.clusterService = clusterService; + this.serviceInfoProvider = serviceInfoProvider; + this.actorContext = actorContext; + } + + @Autowired(required = false) + public void setTbRuleEngineRpcService(Optional tbRuleEngineRpcService) { + this.tbRuleEngineRpcService = tbRuleEngineRpcService; + } + + @PostConstruct + public void initExecutor() { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-core-rpc-scheduler")); + serviceId = serviceInfoProvider.getServiceId(); + } + + @PreDestroy + public void shutdownExecutor() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToRuleEngineRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToRuleEngine(request); + scheduleToRuleEngineTimeout(request, requestId); + } + + @Override + public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), response); + UUID requestId = response.getId(); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + @Override + public void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg rpcMsg) { + ToDeviceRpcRequest request = rpcMsg.getMsg(); + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToDeviceRpcRequests.put(requestId, rpcMsg); + actorContext.tell(rpcMsg, ActorRef.noSender()); + scheduleToDeviceTimeout(request, requestId); + } + + @Override + public void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); + UUID requestId = response.getId(); + ToDeviceRpcRequestActorMsg request = localToDeviceRpcRequests.remove(requestId); + if (request != null) { + sendRpcResponseToTbRuleEngine(request.getServiceId(), response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + private void sendRpcResponseToTbRuleEngine(String originServiceId, FromDeviceRpcResponse response) { + if (serviceId.equals(originServiceId)) { + if (tbRuleEngineRpcService.isPresent()) { + tbRuleEngineRpcService.get().processRpcResponseFromDevice(response); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + clusterService.pushNotificationToRuleEngine(originServiceId, response, null); + } + } + + private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { + ObjectNode entityNode = json.createObjectNode(); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("requestUUID", msg.getId().toString()); + metaData.putValue("originServiceId", serviceId); + metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); + metaData.putValue("oneway", Boolean.toString(msg.isOneway())); + + Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); + if (device != null) { + metaData.putValue("deviceName", device.getName()); + metaData.putValue("deviceType", device.getType()); + } + + entityNode.put("method", msg.getBody().getMethod()); + entityNode.put("params", msg.getBody().getParams()); + + try { + TbMsg tbMsg = TbMsg.newMsg(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); + clusterService.pushMsgToRuleEngine(msg.getTenantId(), msg.getDeviceId(), tbMsg, null); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing to rule engine request.", requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout for processing to rule engine request.", requestId); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); + } + }, timeout, TimeUnit.MILLISECONDS); + } + + private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing to device request.", requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout for to device request.", requestId); + localToDeviceRpcRequests.remove(requestId); + }, timeout, TimeUnit.MILLISECONDS); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java new file mode 100644 index 0000000000..0ec730b7dc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -0,0 +1,177 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; +import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.queue.TbClusterService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +@Service +@TbRuleEngineComponent +@Slf4j +public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcService { + + private final PartitionService partitionService; + private final TbClusterService clusterService; + private final TbServiceInfoProvider serviceInfoProvider; + + + private final ConcurrentMap> toDeviceRpcRequests = new ConcurrentHashMap<>(); + + private Optional tbCoreRpcService; + private ScheduledExecutorService scheduler; + private String serviceId; + + public DefaultTbRuleEngineRpcService(PartitionService partitionService, + TbClusterService clusterService, + TbServiceInfoProvider serviceInfoProvider) { + this.partitionService = partitionService; + this.clusterService = clusterService; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Autowired(required = false) + public void setTbCoreRpcService(Optional tbCoreRpcService) { + this.tbCoreRpcService = tbCoreRpcService; + } + + @PostConstruct + public void initExecutor() { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rule-engine-rpc-scheduler")); + serviceId = serviceInfoProvider.getServiceId(); + } + + @PreDestroy + public void shutdownExecutor() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body) { + TransportProtos.ToServerRpcResponseMsg responseMsg = TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(requestId) + .setPayload(body).build(); + TransportProtos.ToTransportMsg msg = TransportProtos.ToTransportMsg.newBuilder() + .setSessionIdMSB(sessionId.getMostSignificantBits()) + .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setToServerResponse(responseMsg) + .build(); + clusterService.pushNotificationToTransport(serviceId, msg, null); + } + + @Override + public void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest src, Consumer