diff --git a/application/src/main/data/json/demo/dashboards/gateways.json b/application/src/main/data/json/demo/dashboards/gateways.json deleted file mode 100644 index f6370696c1..0000000000 --- a/application/src/main/data/json/demo/dashboards/gateways.json +++ /dev/null @@ -1,1274 +0,0 @@ -{ - "title": "Gateways", - "configuration": { - "widgets": { - "94715984-ae74-76e4-20b7-2f956b01ed80": { - "isSystemType": true, - "bundleAlias": "entity_admin_widgets", - "typeAlias": "device_admin_table2", - "type": "latest", - "title": "New widget", - "sizeX": 24, - "sizeY": 12, - "config": { - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 86400000 - }, - "aggregation": { - "type": "NONE", - "limit": 200 - } - }, - "showTitle": true, - "backgroundColor": "rgb(255, 255, 255)", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "4px", - "settings": { - "enableSearch": true, - "displayPagination": true, - "defaultPageSize": 10, - "defaultSortOrder": "entityName", - "displayEntityName": true, - "displayEntityType": false, - "entitiesTitle": "List of gateways", - "enableSelectColumnDisplay": true, - "displayEntityLabel": false, - "entityNameColumnTitle": "Gateway Name" - }, - "title": "Devices gateway table", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400, - "padding": "5px 10px 5px 10px" - }, - "useDashboardTimewindow": false, - "showLegend": false, - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "active", - "type": "attribute", - "label": "Active", - "color": "#2196f3", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": true, - "useCellContentFunction": true, - "cellContentFunction": "value = '⬤';\nreturn value;", - "cellStyleFunction": "var color;\nif (value == 'false') {\n color = '#EB5757';\n} else {\n color = '#27AE60';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" - }, - "_hash": 0.3646047595211721 - }, - { - "name": "eventsSent", - "type": "timeseries", - "label": "Sent", - "color": "#4caf50", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.7235710720767985 - }, - { - "name": "eventsProduced", - "type": "timeseries", - "label": "Events", - "color": "#f44336", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.5085933386303254 - }, - { - "name": "LOGS", - "type": "timeseries", - "label": "Latest log", - "color": "#ffc107", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.3504240371585048, - "postFuncBody": "if(value) {\n return value.substring(0, 31) + \"...\";\n} else {\n return '';\n}" - }, - { - "name": "RemoteLoggingLevel", - "type": "attribute", - "label": "Log level", - "color": "#607d8b", - "settings": { - "columnWidth": "0px", - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.9785994222542516 - } - ], - "entityAliasId": "3e0f533a-0db1-3292-184f-06e73535061a" - } - ], - "showTitleIcon": true, - "titleIcon": "list", - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "List device", - "widgetStyle": {}, - "displayTimewindow": true, - "actions": { - "headerButton": [ - { - "id": "70837a9d-c3de-a9a7-03c5-dccd14998758", - "name": "Add device", - "icon": "add", - "type": "customPretty", - "customHtml": "\n
\n \n
\n

Add device

\n \n \n \n \n
\n
\n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n
\n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n
\n \n Create\n Cancel\n \n
\n
\n", - "customCss": "", - "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddDeviceDialog();\n\nfunction openAddDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope', '$mdDialog',\n AddDeviceDialogController\n ],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddDeviceDialogController($scope, $mdDialog) {\n let vm = this;\n vm.types = types;\n vm.attributes = {};\n vm.deviceType = \"gateway\";\n\n vm.cancel = () => {\n $mdDialog.hide();\n };\n\n vm.save = () => {\n vm.loading = true;\n $scope.addDeviceForm.$setPristine();\n let device = {\n additionalInfo: {gateway: true},\n name: vm.deviceName,\n type: vm.deviceType,\n label: vm.deviceLabel\n };\n deviceService.saveDevice(device).then(\n (device) => {\n saveAttributes(device.id).then(\n () => {\n vm.loading = false;\n updateAliasData();\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n };\n\n function saveAttributes(entityId) {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({\n key: key,\n value: vm.attributes[key]\n });\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(\n entityId.entityType, entityId.id,\n \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n\n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController\n .resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController\n .setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController\n .getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast(\n 'widgetForceReInit');\n });\n }\n}" - } - ], - "actionCellButton": [ - { - "id": "78845501-234e-a452-6819-82b5b776e99f", - "name": "Configuration", - "icon": "settings", - "type": "openDashboardState", - "targetDashboardStateId": "__entityname__config", - "openRightLayout": false, - "setEntityId": true - }, - { - "id": "f6ffdba8-e40f-2b8d-851b-f5ecaf18606b", - "name": "Graphs", - "icon": "show_chart", - "type": "openDashboardState", - "targetDashboardStateId": "__entityname_grafic", - "setEntityId": true - }, - { - "id": "242671f3-76c6-6982-7acc-6f12addf0ccc", - "name": "Edit device", - "icon": "edit", - "type": "customPretty", - "customHtml": "\n
\n \n
\n

Edit device

\n \n \n \n \n
\n
\n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n
\n \n Create\n Cancel\n \n
\n
", - "customCss": "/*=======================================================================*/\n/*========== There are two examples: for edit and add entity ==========*/\n/*=======================================================================*/\n/*======================== Edit entity example ========================*/\n/*=======================================================================*/\n/*\n.edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n\n.relations-list.old-relations tb-entity-select tb-entity-autocomplete button {\n display: none;\n} \n*/\n/*========================================================================*/\n/*========================= Add entity example =========================*/\n/*========================================================================*/\n/*\n.add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n*/\n", - "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n \nopenEditDeviceDialog();\n\nfunction openEditDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', EditDeviceDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction EditDeviceDialogController($scope,$mdDialog) {\n let vm = this;\n vm.types = types;\n vm.loading = false;\n vm.attributes = {};\n \n getEntityInfo();\n \n function getEntityInfo() {\n vm.loading = true;\n deviceService.getDevice(entityId.id).then(\n (device) => {\n attributeService.getEntityAttributesValues(entityId.entityType, entityId.id, 'SERVER_SCOPE').then(\n (data) => {\n if (data.length) {\n getEntityAttributes(data);\n }\n vm.device = device;\n vm.loading = false;\n } \n );\n }\n )\n }\n \n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n vm.save = () => {\n vm.loading = true;\n $scope.editDeviceForm.$setPristine();\n deviceService.saveDevice(vm.device).then(\n () => {\n saveAttributes().then(\n () => {\n updateAliasData();\n vm.loading = false;\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n }\n \n function getEntityAttributes(attributes) {\n for (let i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n }\n \n function saveAttributes() {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n \n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n console.log(widgetContext);\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}\n" - }, - { - "id": "862ec2b7-fbcf-376e-f85f-b77c07f36efa", - "name": "Delete device", - "icon": "delete", - "type": "custom", - "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteDeviceDialog();\n\nfunction openDeleteDeviceDialog() {\n let title = \"Are you sure you want to delete the device \" + entityName + \"?\";\n let content = \"Be careful, after the confirmation, the device and all related data will become unrecoverable!\";\n let confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(() => {\n deleteDevice();\n })\n}\n\nfunction deleteDevice() {\n deviceService.deleteDevice(entityId.id).then(\n () => {\n updateAliasData();\n }\n );\n}\n\nfunction updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}" - } - ], - "rowClick": [ - { - "id": "ad5fc7e1-5e60-e056-6940-a75a383466a1", - "name": "to_entityname__config", - "icon": "more_horiz", - "type": "openDashboardState", - "targetDashboardStateId": "__entityname__config", - "setEntityId": true, - "stateEntityParamName": "" - } - ] - } - }, - "id": "94715984-ae74-76e4-20b7-2f956b01ed80" - }, - "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { - "isSystemType": true, - "bundleAlias": "cards", - "typeAlias": "timeseries_table", - "type": "timeseries", - "title": "New widget", - "sizeX": 14, - "sizeY": 13, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "LOGS", - "type": "timeseries", - "label": "LOGS", - "color": "#2196f3", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.3496649158709739, - "postFuncBody": "return value.replace(/ - (.*) - \\[/gi, ' - $1 - [');" - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 2592000000 - }, - "aggregation": { - "type": "NONE", - "limit": 200 - } - }, - "showTitle": true, - "backgroundColor": "rgb(255, 255, 255)", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "showTimestamp": true, - "displayPagination": true, - "defaultPageSize": 10 - }, - "title": "Debug events (logs)", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": false, - "showLegend": false, - "widgetStyle": {}, - "actions": {}, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true - }, - "id": "eadabbc7-519e-76fc-ba10-b3fe8c18da10" - }, - "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { - "isSystemType": true, - "bundleAlias": "charts", - "typeAlias": "basic_timeseries", - "type": "timeseries", - "title": "New widget", - "sizeX": 17, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "opcuaEventsProduced", - "type": "timeseries", - "label": "opcuaEventsProduced", - "color": "#2196f3", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.1477920581839779 - }, - { - "name": "opcuaEventsSent", - "type": "timeseries", - "label": "opcuaEventsSent", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.6500957113784758 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 120000 - }, - "aggregation": { - "type": "NONE", - "limit": 25000 - }, - "hideInterval": false, - "hideAggregation": false - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - } - }, - "title": "Real time information", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": false, - "displayTimewindow": true, - "showLegend": true, - "legendConfig": { - "direction": "column", - "position": "right", - "showMin": true, - "showMax": true, - "showAvg": true, - "showTotal": true - }, - "actions": {} - }, - "id": "f928afc4-30d1-8d0c-e3cf-777f9f9d1155" - }, - "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { - "isSystemType": true, - "bundleAlias": "cards", - "typeAlias": "timeseries_table", - "type": "timeseries", - "title": "New widget", - "sizeX": 7, - "sizeY": 7, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "eventsSent", - "type": "timeseries", - "label": "Events", - "color": "#2196f3", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.8156044798125357 - }, - { - "name": "eventsProduced", - "type": "timeseries", - "label": "Produced", - "color": "#4caf50", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": false - }, - "_hash": 0.6538259344015449 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 604800000 - }, - "aggregation": { - "type": "NONE", - "limit": 200 - } - }, - "showTitle": true, - "backgroundColor": "rgb(255, 255, 255)", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "showTimestamp": true, - "displayPagination": true, - "defaultPageSize": 6, - "hideEmptyLines": true - }, - "title": "Total Messages", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": false, - "showLegend": false, - "widgetStyle": {}, - "actions": {}, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true, - "legendConfig": { - "direction": "column", - "position": "bottom", - "showMin": false, - "showMax": false, - "showAvg": true, - "showTotal": false - } - }, - "id": "2a95b473-042d-59d0-2da2-40d0cccb6c8a" - }, - "aaa69366-aacc-9028-65aa-645c0f8533ec": { - "isSystemType": true, - "bundleAlias": "charts", - "typeAlias": "basic_timeseries", - "type": "timeseries", - "title": "New widget", - "sizeX": 17, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "eventsSent", - "type": "timeseries", - "label": "eventsSent", - "color": "#2196f3", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.41414001784591314 - }, - { - "name": "eventsProduced", - "type": "timeseries", - "label": "eventsProduced", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.7819101846284422 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - } - }, - "title": "History information", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": true, - "displayTimewindow": true, - "showLegend": true, - "legendConfig": { - "direction": "column", - "position": "right", - "showMin": true, - "showMax": true, - "showAvg": true, - "showTotal": true - }, - "actions": {} - }, - "id": "aaa69366-aacc-9028-65aa-645c0f8533ec" - }, - "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { - "isSystemType": true, - "bundleAlias": "charts", - "typeAlias": "basic_timeseries", - "type": "timeseries", - "title": "New widget", - "sizeX": 17, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "bleEventsProduced", - "type": "timeseries", - "label": "bleEventsProduced", - "color": "#2196f3", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.5625165504526104 - }, - { - "name": "bleEventsSent", - "type": "timeseries", - "label": "bleEventsSent", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "tooltipValueFormatter": "", - "showSeparateAxis": false, - "axisTitle": "", - "axisPosition": "left", - "axisTicksFormatter": "", - "comparisonSettings": { - "showValuesForComparison": true, - "comparisonValuesLabel": "", - "color": "" - } - }, - "_hash": 0.6817950080745288 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "interval": 5000, - "timewindowMs": 120000 - }, - "aggregation": { - "type": "AVG", - "limit": 25000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - } - }, - "title": "Real time information", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": false, - "displayTimewindow": true, - "showLegend": true, - "legendConfig": { - "direction": "column", - "position": "right", - "showMin": true, - "showMax": true, - "showAvg": true, - "showTotal": true - }, - "actions": {} - }, - "id": "ce5c7d01-a3ef-5cf0-4578-8505135c23a0" - }, - "466f046d-6005-a168-b107-60fcb2469cd5": { - "isSystemType": true, - "bundleAlias": "gateway_widgets", - "typeAlias": "attributes_card", - "type": "latest", - "title": "New widget", - "sizeX": 7, - "sizeY": 5, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "eventsTitle": "Gateway Events Form", - "eventsReg": [ - "EventsProduced", - "EventsSent" - ] - }, - "title": "Gateway events", - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "dropShadow": true, - "enableFullscreen": true, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "displayTimewindow": true, - "showLegend": false, - "actions": {} - }, - "id": "466f046d-6005-a168-b107-60fcb2469cd5" - }, - "8fc32225-164f-3258-73f7-e6b6d959cf0b": { - "isSystemType": true, - "bundleAlias": "gateway_widgets", - "typeAlias": "config_form_latest", - "type": "latest", - "title": "New widget", - "sizeX": 10, - "sizeY": 9, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "gatewayTitle": "Gateway configuration (Single device)", - "readOnly": false - }, - "title": "New Gateway configuration (Single device)", - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "dropShadow": true, - "enableFullscreen": true, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "displayTimewindow": true, - "showLegend": false, - "actions": {} - }, - "id": "8fc32225-164f-3258-73f7-e6b6d959cf0b" - }, - "063fc179-c9fd-f952-e714-f24e9c43c05c": { - "isSystemType": true, - "bundleAlias": "control_widgets", - "typeAlias": "rpcbutton", - "type": "rpc", - "title": "New widget", - "sizeX": 4, - "sizeY": 2, - "config": { - "targetDeviceAliases": [], - "showTitle": false, - "backgroundColor": "#e6e7e8", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "0px", - "settings": { - "requestTimeout": 5000, - "oneWayElseTwoWay": true, - "styleButton": { - "isRaised": true, - "isPrimary": false - }, - "methodParams": "{}", - "methodName": "gateway_reboot", - "buttonText": "GATEWAY REBOOT" - }, - "title": "New RPC Button", - "dropShadow": true, - "enableFullscreen": false, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "showLegend": false, - "actions": {}, - "datasources": [], - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true, - "targetDeviceAliasIds": [ - "b2487e75-2fa4-f211-142c-434dfd50c70c" - ] - }, - "id": "063fc179-c9fd-f952-e714-f24e9c43c05c" - }, - "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { - "isSystemType": true, - "bundleAlias": "control_widgets", - "typeAlias": "rpcbutton", - "type": "rpc", - "title": "New widget", - "sizeX": 4, - "sizeY": 2, - "config": { - "targetDeviceAliases": [], - "showTitle": false, - "backgroundColor": "#e6e7e8", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "0px", - "settings": { - "requestTimeout": 5000, - "oneWayElseTwoWay": true, - "styleButton": { - "isRaised": true, - "isPrimary": false - }, - "methodName": "gateway_restart", - "methodParams": "{}", - "buttonText": "gateway restart" - }, - "title": "New RPC Button", - "dropShadow": true, - "enableFullscreen": false, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "showLegend": false, - "actions": {}, - "datasources": [], - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true, - "targetDeviceAliasIds": [ - "b2487e75-2fa4-f211-142c-434dfd50c70c" - ] - }, - "id": "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7" - }, - "6770b6ba-eff8-df05-75f8-c1f9326d4842": { - "isSystemType": true, - "bundleAlias": "input_widgets", - "typeAlias": "markers_placement_openstreetmap", - "type": "latest", - "title": "New widget", - "sizeX": 6, - "sizeY": 4, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "latitude", - "type": "attribute", - "label": "latitude", - "color": "#2196f3", - "settings": {}, - "_hash": 0.9743324774725604 - }, - { - "name": "longitude", - "type": "attribute", - "label": "longitude", - "color": "#4caf50", - "settings": {}, - "_hash": 0.5530093635101525 - } - ], - "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": false, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "fitMapBounds": true, - "latKeyName": "latitude", - "lngKeyName": "longitude", - "showLabel": true, - "label": "${entityName}", - "tooltipPattern": "${entityName}

Latitude: ${latitude:7}
Longitude: ${longitude:7}

Delete", - "markerImageSize": 34, - "useColorFunction": false, - "markerImages": [], - "useMarkerImageFunction": false, - "color": "#fe7569", - "mapProvider": "OpenStreetMap.Mapnik", - "showTooltip": true, - "autocloseTooltip": true, - "defaultCenterPosition": [ - 0, - 0 - ], - "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", - "showTooltipAction": "click", - "polygonKeyName": "coordinates", - "polygonOpacity": 0.5, - "polygonStrokeOpacity": 1, - "polygonStrokeWeight": 1, - "zoomOnClick": true, - "showCoverageOnHover": true, - "animate": true, - "maxClusterRadius": 80, - "removeOutsideVisibleBounds": true, - "defaultZoomLevel": 5 - }, - "title": "Gateway Location", - "dropShadow": true, - "enableFullscreen": false, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "showLegend": false, - "widgetStyle": {}, - "actions": { - "tooltipAction": [ - { - "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", - "name": "delete", - "icon": "more_horiz", - "type": "custom", - "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" - } - ] - }, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true - }, - "id": "6770b6ba-eff8-df05-75f8-c1f9326d4842" - } - }, - "states": { - "main_gateway": { - "name": "Gateways", - "root": true, - "layouts": { - "main": { - "widgets": { - "94715984-ae74-76e4-20b7-2f956b01ed80": { - "sizeX": 24, - "sizeY": 12, - "row": 0, - "col": 0 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "margins": [ - 10, - 10 - ], - "backgroundSizeMode": "100%", - "autoFillHeight": true, - "mobileAutoFillHeight": false, - "mobileRowHeight": 70 - } - } - } - }, - "__entityname__config": { - "name": "${entityName} Configuration", - "root": false, - "layouts": { - "main": { - "widgets": { - "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { - "sizeX": 14, - "sizeY": 13, - "row": 0, - "col": 10 - }, - "8fc32225-164f-3258-73f7-e6b6d959cf0b": { - "sizeX": 10, - "sizeY": 9, - "row": 0, - "col": 0 - }, - "063fc179-c9fd-f952-e714-f24e9c43c05c": { - "sizeX": 4, - "sizeY": 2, - "row": 9, - "col": 0 - }, - "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { - "sizeX": 4, - "sizeY": 2, - "row": 11, - "col": 0 - }, - "6770b6ba-eff8-df05-75f8-c1f9326d4842": { - "sizeX": 6, - "sizeY": 4, - "row": 9, - "col": 4 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "margins": [ - 10, - 10 - ], - "backgroundSizeMode": "100%", - "autoFillHeight": true, - "mobileAutoFillHeight": false, - "mobileRowHeight": 70 - } - } - } - }, - "__entityname_grafic": { - "name": "${entityName} Details", - "root": false, - "layouts": { - "main": { - "widgets": { - "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { - "sizeX": 17, - "sizeY": 4, - "mobileHeight": null, - "row": 4, - "col": 7 - }, - "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { - "sizeX": 7, - "sizeY": 7, - "row": 5, - "col": 0 - }, - "aaa69366-aacc-9028-65aa-645c0f8533ec": { - "sizeX": 17, - "sizeY": 4, - "mobileHeight": null, - "row": 0, - "col": 7 - }, - "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { - "sizeX": 17, - "sizeY": 4, - "mobileHeight": null, - "row": 8, - "col": 7 - }, - "466f046d-6005-a168-b107-60fcb2469cd5": { - "sizeX": 7, - "sizeY": 5, - "row": 0, - "col": 0 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "margins": [ - 10, - 10 - ], - "backgroundSizeMode": "auto 100%", - "autoFillHeight": true, - "mobileAutoFillHeight": true, - "mobileRowHeight": 70 - } - } - } - } - }, - "entityAliases": { - "3e0f533a-0db1-3292-184f-06e73535061a": { - "id": "3e0f533a-0db1-3292-184f-06e73535061a", - "alias": "Gateways", - "filter": { - "type": "deviceType", - "resolveMultiple": true, - "deviceType": "gateway", - "deviceNameFilter": "" - } - }, - "b2487e75-2fa4-f211-142c-434dfd50c70c": { - "id": "b2487e75-2fa4-f211-142c-434dfd50c70c", - "alias": "Current Gateway", - "filter": { - "type": "stateEntity", - "resolveMultiple": false, - "stateEntityParamName": "", - "defaultStateEntity": null - } - } - }, - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 86400000 - }, - "aggregation": { - "type": "NONE", - "limit": 25000 - }, - "hideInterval": false, - "hideAggregation": false, - "hideAggInterval": false - }, - "settings": { - "stateControllerId": "entity", - "showTitle": true, - "showDashboardsSelect": true, - "showEntitiesSelect": true, - "showDashboardTimewindow": true, - "showDashboardExport": true, - "toolbarAlwaysOpen": true, - "titleColor": "rgba(0,0,0,0.870588)" - } - }, - "name": "Gateways" -} \ No newline at end of file diff --git a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json deleted file mode 100644 index b8231ab880..0000000000 --- a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json +++ /dev/null @@ -1,520 +0,0 @@ -{ - "title": "Rule Engine Statistics", - "configuration": { - "widgets": { - "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { - "isSystemType": true, - "bundleAlias": "charts", - "typeAlias": "basic_timeseries", - "type": "timeseries", - "title": "New widget", - "sizeX": 12, - "sizeY": 7, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "successfulMsgs", - "type": "timeseries", - "label": "${entityName} Successful", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "showSeparateAxis": false, - "axisPosition": "left", - "thresholds": [ - { - "thresholdValueSource": "predefinedValue" - } - ], - "comparisonSettings": { - "showValuesForComparison": true - } - }, - "_hash": 0.15490750967648736 - }, - { - "name": "failedMsgs", - "type": "timeseries", - "label": "${entityName} Permanent Failures", - "color": "#ef5350", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "showSeparateAxis": false, - "axisPosition": "left", - "thresholds": [ - { - "thresholdValueSource": "predefinedValue" - } - ], - "comparisonSettings": { - "showValuesForComparison": true - } - }, - "_hash": 0.4186621166514697 - }, - { - "name": "tmpFailed", - "type": "timeseries", - "label": "${entityName} Processing Failures", - "color": "#ffc107", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "showSeparateAxis": false, - "axisPosition": "left", - "thresholds": [ - { - "thresholdValueSource": "predefinedValue" - } - ], - "comparisonSettings": { - "showValuesForComparison": true - } - }, - "_hash": 0.49891007198715376 - } - ], - "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 300000 - }, - "aggregation": { - "type": "NONE", - "limit": 8640 - }, - "hideInterval": false, - "hideAggregation": false, - "hideAggInterval": false - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - } - }, - "title": "Queue Stats", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": false, - "displayTimewindow": true, - "showLegend": true, - "actions": {}, - "legendConfig": { - "direction": "column", - "position": "bottom", - "showMin": true, - "showMax": true, - "showAvg": false, - "showTotal": true - } - }, - "id": "81987f19-3eac-e4ce-b790-d96e9b54d9a0" - }, - "5eb79712-5c24-3060-7e4f-6af36b8f842d": { - "isSystemType": true, - "bundleAlias": "cards", - "typeAlias": "timeseries_table", - "type": "timeseries", - "title": "New widget", - "sizeX": 24, - "sizeY": 5, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "ruleEngineException", - "type": "timeseries", - "label": "Rule Chain", - "color": "#2196f3", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": true, - "cellContentFunction": "return JSON.parse(value).ruleChainName;" - }, - "_hash": 0.9954481282345906 - }, - { - "name": "ruleEngineException", - "type": "timeseries", - "label": "Rule Node", - "color": "#4caf50", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": true, - "cellContentFunction": "return JSON.parse(value).ruleNodeName;" - }, - "_hash": 0.18580357036589978 - }, - { - "name": "ruleEngineException", - "type": "timeseries", - "label": "Latest Error", - "color": "#f44336", - "settings": { - "useCellStyleFunction": false, - "useCellContentFunction": true, - "cellContentFunction": "return JSON.parse(value).message;" - }, - "_hash": 0.7255162989552142 - } - ], - "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 86400000 - }, - "aggregation": { - "type": "NONE", - "limit": 200 - } - }, - "showTitle": true, - "backgroundColor": "rgb(255, 255, 255)", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "showTimestamp": true, - "displayPagination": true, - "defaultPageSize": 10 - }, - "title": "Exceptions", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": false, - "showLegend": false, - "widgetStyle": {}, - "actions": {}, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "displayTimewindow": true - }, - "id": "5eb79712-5c24-3060-7e4f-6af36b8f842d" - }, - "ad3f1417-87a8-750e-fc67-49a2de1466d4": { - "isSystemType": true, - "bundleAlias": "charts", - "typeAlias": "basic_timeseries", - "type": "timeseries", - "title": "New widget", - "sizeX": 12, - "sizeY": 7, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "timeoutMsgs", - "type": "timeseries", - "label": "${entityName} Permanent Timeouts", - "color": "#4caf50", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "showSeparateAxis": false, - "axisPosition": "left", - "thresholds": [ - { - "thresholdValueSource": "predefinedValue" - } - ], - "comparisonSettings": { - "showValuesForComparison": true - } - }, - "_hash": 0.565222981550328 - }, - { - "name": "tmpTimeout", - "type": "timeseries", - "label": "${entityName} Processing Timeouts", - "color": "#9c27b0", - "settings": { - "excludeFromStacking": false, - "hideDataByDefault": false, - "disableDataHiding": false, - "removeFromLegend": false, - "showLines": true, - "fillLines": false, - "showPoints": false, - "showPointShape": "circle", - "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", - "showPointsLineWidth": 5, - "showPointsRadius": 3, - "showSeparateAxis": false, - "axisPosition": "left", - "thresholds": [ - { - "thresholdValueSource": "predefinedValue" - } - ], - "comparisonSettings": { - "showValuesForComparison": true - } - }, - "_hash": 0.2679547062508352 - } - ], - "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" - } - ], - "timewindow": { - "realtime": { - "interval": 1000, - "timewindowMs": 300000 - }, - "aggregation": { - "type": "NONE", - "limit": 8640 - }, - "hideInterval": false, - "hideAggregation": false, - "hideAggInterval": false - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "shadowSize": 4, - "fontColor": "#545454", - "fontSize": 10, - "xaxis": { - "showLabels": true, - "color": "#545454" - }, - "yaxis": { - "showLabels": true, - "color": "#545454" - }, - "grid": { - "color": "#545454", - "tickColor": "#DDDDDD", - "verticalLines": true, - "horizontalLines": true, - "outlineWidth": 1 - }, - "stack": false, - "tooltipIndividual": false, - "timeForComparison": "months", - "xaxisSecond": { - "axisPosition": "top", - "showLabels": true - } - }, - "title": "Processing Failures and Timeouts", - "dropShadow": true, - "enableFullscreen": true, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "mobileHeight": null, - "showTitleIcon": false, - "titleIcon": null, - "iconColor": "rgba(0, 0, 0, 0.87)", - "iconSize": "24px", - "titleTooltip": "", - "widgetStyle": {}, - "useDashboardTimewindow": false, - "displayTimewindow": true, - "showLegend": true, - "actions": {}, - "legendConfig": { - "direction": "column", - "position": "bottom", - "showMin": true, - "showMax": true, - "showAvg": false, - "showTotal": true - } - }, - "id": "ad3f1417-87a8-750e-fc67-49a2de1466d4" - } - }, - "states": { - "default": { - "name": "Rule Engine Statistics", - "root": true, - "layouts": { - "main": { - "widgets": { - "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { - "sizeX": 12, - "sizeY": 7, - "mobileHeight": null, - "row": 0, - "col": 0 - }, - "5eb79712-5c24-3060-7e4f-6af36b8f842d": { - "sizeX": 24, - "sizeY": 5, - "row": 7, - "col": 0 - }, - "ad3f1417-87a8-750e-fc67-49a2de1466d4": { - "sizeX": 12, - "sizeY": 7, - "mobileHeight": null, - "row": 0, - "col": 12 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "margins": [ - 10, - 10 - ], - "backgroundSizeMode": "100%", - "autoFillHeight": true, - "mobileAutoFillHeight": false, - "mobileRowHeight": 70 - } - } - } - } - }, - "entityAliases": { - "140f23dd-e3a0-ed98-6189-03c49d2d8018": { - "id": "140f23dd-e3a0-ed98-6189-03c49d2d8018", - "alias": "TbServiceQueues", - "filter": { - "type": "assetType", - "resolveMultiple": true, - "assetType": "TbServiceQueue", - "assetNameFilter": "" - } - } - }, - "timewindow": { - "displayValue": "", - "selectedTab": 0, - "hideInterval": false, - "hideAggregation": false, - "hideAggInterval": false, - "realtime": { - "interval": 1000, - "timewindowMs": 60000 - }, - "history": { - "historyType": 0, - "interval": 1000, - "timewindowMs": 60000, - "fixedTimewindow": { - "startTimeMs": 1586176634823, - "endTimeMs": 1586263034823 - } - }, - "aggregation": { - "type": "AVG", - "limit": 25000 - } - }, - "settings": { - "stateControllerId": "entity", - "showTitle": false, - "showDashboardsSelect": true, - "showEntitiesSelect": true, - "showDashboardTimewindow": true, - "showDashboardExport": true, - "toolbarAlwaysOpen": true - } - }, - "name": "Rule Engine Statistics" -} \ No newline at end of file diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql index dda3bd7b1c..ff1fb5129b 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -123,28 +123,3 @@ BEGIN END LOOP; END $$; - -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - ttl_ts bigint; - debug_ttl_ts bigint; - ttl_deleted_count bigint DEFAULT 0; - debug_ttl_deleted_count bigint DEFAULT 0; -BEGIN - IF ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; - END IF; - IF debug_ttl > 0 THEN - debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; - END IF; - RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; - RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; - deleted := ttl_deleted_count + debug_ttl_deleted_count; -END -$$; diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 7e9b68882b..54ac41172c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -32,10 +32,12 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.common.data.DataConstants; @@ -43,11 +45,10 @@ import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -64,23 +65,24 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.service.cluster.discovery.DiscoveryService; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; import org.thingsboard.server.service.mail.MailExecutorService; -import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; -import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; +import org.thingsboard.server.service.rpc.DeviceRpcService; import org.thingsboard.server.service.script.JsExecutorService; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.session.DeviceSessionCacheService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.service.transport.TbCoreToTransportService; +import org.thingsboard.server.service.transport.RuleEngineTransportService; import javax.annotation.Nullable; import java.io.IOException; @@ -104,24 +106,35 @@ public class ActorSystemContext { return debugPerTenantLimits; } - @Autowired @Getter @Setter - private TbServiceInfoProvider serviceInfoProvider; + private ActorService actorService; + @Autowired @Getter - @Setter - private ActorService actorService; + private DiscoveryService discoveryService; @Autowired @Getter @Setter private ComponentDiscoveryService componentService; + @Autowired + @Getter + private ClusterRoutingService routingService; + + @Autowired + @Getter + private ClusterRpcService rpcService; + @Autowired @Getter private DataDecodingEncodingService encodingService; + @Autowired + @Getter + private DeviceAuthService deviceAuthService; + @Autowired @Getter private DeviceService deviceService; @@ -150,13 +163,6 @@ public class ActorSystemContext { @Getter private RuleChainService ruleChainService; - @Autowired - private PartitionService partitionService; - - @Autowired - @Getter - private TbClusterService clusterService; - @Autowired @Getter private TimeseriesService tsService; @@ -189,6 +195,10 @@ public class ActorSystemContext { @Getter private TelemetrySubscriptionService tsSubService; + @Autowired + @Getter + private DeviceRpcService deviceRpcService; + @Autowired @Getter private JsInvokeService jsSandbox; @@ -201,6 +211,10 @@ public class ActorSystemContext { @Getter private MailExecutorService mailExecutor; + @Autowired + @Getter + private ClusterRpcCallbackExecutorService clusterRpcCallbackExecutor; + @Autowired @Getter private DbCallbackExecutorService dbCallbackExecutor; @@ -217,32 +231,27 @@ public class ActorSystemContext { @Getter private MailService mailService; - //TODO: separate context for TbCore and TbRuleEngine - @Autowired(required = false) + @Autowired @Getter private DeviceStateService deviceStateService; - @Autowired(required = false) + @Autowired @Getter private DeviceSessionCacheService deviceSessionCacheService; - @Autowired(required = false) + @Lazy + @Autowired @Getter - private TbCoreToTransportService tbCoreToTransportService; + private RuleEngineTransportService ruleEngineTransportService; - /** - * The following Service will be null if we operate in tb-core mode - */ - @Autowired(required = false) + @Lazy + @Autowired @Getter - private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; + private RuleChainTransactionService ruleChainTransactionService; - /** - * The following Service will be null if we operate in tb-rule-engine mode - */ - @Autowired(required = false) + @Value("${cluster.partition_id}") @Getter - private TbCoreDeviceRpcService tbCoreDeviceRpcService; + private long queuePartitionId; @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter @@ -260,6 +269,10 @@ public class ActorSystemContext { @Getter private long queuePersistenceTimeout; + @Value("${actors.client_side_rpc.timeout}") + @Getter + private long clientSideRpcTimeout; + @Value("${actors.rule.chain.error_persist_frequency}") @Getter private long ruleChainErrorPersistFrequency; @@ -321,6 +334,11 @@ public class ActorSystemContext { @Setter private ActorSystem actorSystem; + @Autowired + @Getter + private TbNodeIdProvider nodeIdProvider; + + @Getter @Setter private ActorRef appActor; @@ -347,8 +365,6 @@ public class ActorSystemContext { config = ConfigFactory.parseResources(AKKA_CONF_FILE_NAME).withFallback(ConfigFactory.load()); } - - public Scheduler getScheduler() { return actorSystem.scheduler(); } @@ -358,7 +374,7 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.ERROR); - event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e))); + event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); persistEvent(event); } @@ -367,7 +383,7 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); - event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e))); + event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -381,8 +397,8 @@ public class ActorSystemContext { return sw.toString(); } - private JsonNode toBodyJson(String serviceId, ComponentLifecycleEvent event, Optional e) { - ObjectNode node = mapper.createObjectNode().put("server", serviceId).put("event", event.name()); + private JsonNode toBodyJson(ServerAddress server, ComponentLifecycleEvent event, Optional e) { + ObjectNode node = mapper.createObjectNode().put("server", server.toString()).put("event", event.name()); if (e.isPresent()) { node = node.put("success", false); node = node.put("error", toString(e.get())); @@ -392,21 +408,12 @@ public class ActorSystemContext { return node; } - private JsonNode toBodyJson(String serviceId, String method, String body) { - return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); + private JsonNode toBodyJson(ServerAddress server, String method, String body) { + return mapper.createObjectNode().put("server", server.toString()).put("method", method).put("error", body); } - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { - return partitionService.resolve(serviceType, tenantId, entityId); - } - - public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { - return partitionService.resolve(serviceType, queueName, tenantId, entityId); - } - - - public String getServiceId() { - return serviceInfoProvider.getServiceId(); + public String getServerAddress() { + return discoveryService.getCurrentServer().getServerAddress().toString(); } public void persistDebugInput(TenantId tenantId, EntityId entityId, TbMsg tbMsg, String relationType) { @@ -437,7 +444,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() .put("type", type) - .put("server", getServiceId()) + .put("server", getServerAddress()) .put("entityId", tbMsg.getOriginator().getId().toString()) .put("entityName", tbMsg.getOriginator().getEntityType().name()) .put("msgId", tbMsg.getId().toString()) @@ -497,7 +504,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() //todo: what fields are needed here? - .put("server", getServiceId()) + .put("server", getServerAddress()) .put("message", "Reached debug mode rate limit!"); if (error != null) { @@ -523,7 +530,4 @@ public class ActorSystemContext { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } - public void tell(TbActorMsg tbActorMsg, ActorRef sender) { - appActor.tell(tbActorMsg, sender); - } } diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index b7782a6a87..778c945a32 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -20,13 +20,19 @@ import akka.actor.LocalActorRef; import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; +import akka.actor.SupervisorStrategy.Directive; import akka.actor.Terminated; +import akka.event.Logging; +import akka.event.LoggingAdapter; +import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.actors.shared.rulechain.SystemRuleChainManager; import org.thingsboard.server.actors.tenant.TenantActor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; @@ -36,32 +42,29 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import scala.concurrent.duration.Duration; -import java.util.HashSet; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; -import java.util.Set; -public class AppActor extends ContextAwareActor { +public class AppActor extends RuleChainManagerActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); private final TenantService tenantService; private final BiMap tenantActors; - private final Set deletedTenants; private boolean ruleChainsInitialized; private AppActor(ActorSystemContext systemContext) { - super(systemContext); + super(systemContext, new SystemRuleChainManager(systemContext)); this.tenantService = systemContext.getTenantService(); this.tenantActors = HashBiMap.create(); - this.deletedTenants = new HashSet<>(); } @Override @@ -76,7 +79,7 @@ public class AppActor extends ContextAwareActor { @Override protected boolean process(TbActorMsg msg) { if (!ruleChainsInitialized) { - initTenantActors(); + initRuleChainsAndTenantActors(); ruleChainsInitialized = true; if (msg.getMsgType() != MsgType.APP_INIT_MSG) { log.warn("Rule Chains initialized by unexpected message: {}", msg); @@ -85,14 +88,17 @@ public class AppActor extends ContextAwareActor { switch (msg.getMsgType()) { case APP_INIT_MSG: break; - case PARTITION_CHANGE_MSG: + case SEND_TO_CLUSTER_MSG: + onPossibleClusterMsg((SendToClusterMsg) msg); + break; + case CLUSTER_EVENT_MSG: broadcast(msg); break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case QUEUE_TO_RULE_ENGINE_MSG: - onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); + case SERVICE_TO_RULE_ENGINE_MSG: + onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -100,6 +106,7 @@ public class AppActor extends ContextAwareActor { case DEVICE_NAME_OR_TYPE_UPDATE_TO_DEVICE_ACTOR_MSG: case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: + case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onToDeviceActorMsg((TenantAwareMsg) msg); break; default: @@ -108,30 +115,16 @@ public class AppActor extends ContextAwareActor { return true; } - private void initTenantActors() { + private void initRuleChainsAndTenantActors() { log.info("Starting main system actor."); try { - // This Service may be started for specific tenant only. - Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); - if (isolatedTenantId.isPresent()) { - Tenant tenant = systemContext.getTenantService().findTenantById(isolatedTenantId.get()); - if (tenant != null) { + initRuleChains(); + if (systemContext.isTenantComponentsInitEnabled()) { + PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); + for (Tenant tenant : tenantIterator) { log.debug("[{}] Creating tenant actor", tenant.getId()); getOrCreateTenantActor(tenant.getId()); log.debug("Tenant actor created."); - } else { - log.error("[{}] Tenant with such ID does not exist", isolatedTenantId.get()); - } - } else if (systemContext.isTenantComponentsInitEnabled()) { - PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); - boolean isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); - boolean isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - for (Tenant tenant : tenantIterator) { - if (isCore || (isRuleEngine && !tenant.isIsolatedTbRuleEngine())) { - log.debug("[{}] Creating tenant actor", tenant.getId()); - getOrCreateTenantActor(tenant.getId()); - log.debug("[{}] Tenant actor created.", tenant.getId()); - } } } log.info("Main system actor started."); @@ -140,33 +133,40 @@ public class AppActor extends ContextAwareActor { } } - private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { + private void onPossibleClusterMsg(SendToClusterMsg msg) { + Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); + if (address.isPresent()) { + systemContext.getRpcService().tell( + systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); + } else { + self().tell(msg.getMsg(), ActorRef.noSender()); + } + } + + private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { if (SYSTEM_TENANT.equals(msg.getTenantId())) { - msg.getTbMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); +// this may be a notification about system entities created. +// log.warn("[{}] Invalid service to rule engine msg called. System messages are not supported yet: {}", SYSTEM_TENANT, msg); } else { - if (!deletedTenants.contains(msg.getTenantId())) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); - } else { - msg.getTbMsg().getCallback().onSuccess(); - } + getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); } } + @Override protected void broadcast(Object msg) { + super.broadcast(msg); tenantActors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { ActorRef target = null; if (SYSTEM_TENANT.equals(msg.getTenantId())) { - log.warn("Message has system tenant id: {}", msg); + target = getEntityActorRef(msg.getEntityId()); } else { if (msg.getEntityId().getEntityType() == EntityType.TENANT && msg.getEvent() == ComponentLifecycleEvent.DELETED) { - log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); - TenantId tenantId = new TenantId(msg.getEntityId().getId()); - deletedTenants.add(tenantId); - ActorRef tenantActor = tenantActors.get(tenantId); + log.debug("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); + ActorRef tenantActor = tenantActors.remove(new TenantId(msg.getEntityId().getId())); if (tenantActor != null) { log.debug("[{}] Deleting tenant actor: {}", msg.getTenantId(), tenantActor); context().stop(tenantActor); @@ -183,22 +183,16 @@ public class AppActor extends ContextAwareActor { } private void onToDeviceActorMsg(TenantAwareMsg msg) { - if (!deletedTenants.contains(msg.getTenantId())) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); - } else { - if (msg instanceof TransportToDeviceActorMsgWrapper) { - ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); - } - } + getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); } private ActorRef getOrCreateTenantActor(TenantId tenantId) { return tenantActors.computeIfAbsent(tenantId, k -> { - log.info("[{}] Creating tenant actor.", tenantId); + log.debug("[{}] Creating tenant actor.", tenantId); ActorRef tenantActor = context().actorOf(Props.create(new TenantActor.ActorCreator(systemContext, tenantId)) .withDispatcher(DefaultActorService.CORE_DISPATCHER_NAME), tenantId.toString()); context().watch(tenantActor); - log.info("[{}] Created tenant actor: {}.", tenantId, tenantActor); + log.debug("[{}] Created tenant actor: {}.", tenantId, tenantActor); return tenantActor; }); } diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java index 256ffd0b30..547e484652 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java @@ -22,8 +22,10 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; +import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; public class DeviceActor extends ContextAwareActor { @@ -46,11 +48,6 @@ public class DeviceActor extends ContextAwareActor { } } - @Override - public void postStop() { - - } - @Override protected boolean process(TbActorMsg msg) { switch (msg.getMsgType()) { @@ -69,9 +66,15 @@ public class DeviceActor extends ContextAwareActor { case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: processor.processRpcRequest(context(), (ToDeviceRpcRequestActorMsg) msg); break; + case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: + processor.processToServerRPCResponse(context(), (ToServerRpcResponseActorMsg) msg); + break; case DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG: processor.processServerSideRpcTimeout(context(), (DeviceActorServerSideRpcTimeoutMsg) msg); break; + case DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG: + processor.processClientSideRpcTimeout(context(), (DeviceActorClientSideRpcTimeoutMsg) msg); + break; case SESSION_TIMEOUT_MSG: processor.checkSessionsTimeout(); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 437ed456e9..2752b53090 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -16,10 +16,13 @@ package org.thingsboard.server.actors.device; import akka.actor.ActorContext; +import com.datastax.driver.core.utils.UUIDs; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import com.google.gson.JsonObject; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -35,34 +38,38 @@ import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.common.msg.session.SessionMsgType; +import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; +import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SessionSubscriptionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SessionType; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; +import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; +import org.thingsboard.server.utils.JsonUtils; import javax.annotation.Nullable; import java.util.ArrayList; @@ -93,6 +100,9 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private final Map attributeSubscriptions; private final Map rpcSubscriptions; private final Map toDeviceRpcPendingMap; + private final Map toServerRpcPendingMap; + + private final Gson gson = new Gson(); private int rpcSeq = 0; private String deviceName; @@ -107,6 +117,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { this.attributeSubscriptions = new HashMap<>(); this.rpcSubscriptions = new HashMap<>(); this.toDeviceRpcPendingMap = new HashMap<>(); + this.toServerRpcPendingMap = new HashMap<>(); if (initAttributes()) { restoreSessions(); } @@ -142,7 +153,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { Set syncSessionSet = new HashSet<>(); rpcSubscriptions.forEach((key, value) -> { sendToTransport(rpcRequest, key, value.getNodeId()); - if (SessionType.SYNC == value.getType()) { + if (TransportProtos.SessionType.SYNC == value.getType()) { syncSessionSet.add(key); } }); @@ -150,7 +161,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (request.isOneway() && sent) { log.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId()); - systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); } else { registerPendingRpcRequest(context, msg, sent, rpcRequest, timeout); } @@ -171,16 +182,16 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(msg.getId()); if (requestMd != null) { log.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId()); - systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION)); } } private void sendPendingRequests(ActorContext context, UUID sessionId, SessionInfoProto sessionInfo) { - SessionType sessionType = getSessionType(sessionId); + TransportProtos.SessionType sessionType = getSessionType(sessionId); if (!toDeviceRpcPendingMap.isEmpty()) { log.debug("[{}] Pushing {} pending RPC messages to new async session [{}]", deviceId, toDeviceRpcPendingMap.size(), sessionId); - if (sessionType == SessionType.SYNC) { + if (sessionType == TransportProtos.SessionType.SYNC) { log.debug("[{}] Cleanup sync rpc session [{}]", deviceId, sessionId); rpcSubscriptions.remove(sessionId); } @@ -188,7 +199,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No pending RPC messages for new async session [{}]", deviceId, sessionId); } Set sentOneWayIds = new HashSet<>(); - if (sessionType == SessionType.ASYNC) { + if (sessionType == TransportProtos.SessionType.ASYNC) { toDeviceRpcPendingMap.entrySet().forEach(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); } else { toDeviceRpcPendingMap.entrySet().stream().findFirst().ifPresent(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); @@ -203,7 +214,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestBody body = request.getBody(); if (request.isOneway()) { sentOneWayIds.add(entry.getKey()); - systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); } ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder().setRequestId( entry.getKey()).setMethodName(body.getMethod()).setParams(body.getParams()).build(); @@ -212,8 +223,8 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { + boolean reportDeviceActivity = false; TransportToDeviceActorMsg msg = wrapper.getMsg(); - TbCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { processSessionStateMsgs(msg.getSessionInfo(), msg.getSessionEvent()); } @@ -223,16 +234,34 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (msg.hasSubscribeToRPC()) { processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToRPC()); } + if (msg.hasPostAttributes()) { + handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes()); + reportDeviceActivity = true; + } + if (msg.hasPostTelemetry()) { + handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry()); + reportDeviceActivity = true; + } if (msg.hasGetAttributes()) { handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes()); } if (msg.hasToDeviceRPCCallResponse()) { processRpcResponses(context, msg.getSessionInfo(), msg.getToDeviceRPCCallResponse()); } + if (msg.hasToServerRPCCallRequest()) { + handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest()); + reportDeviceActivity = true; + } if (msg.hasSubscriptionInfo()) { handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo()); } - callback.onSuccess(); + if (reportDeviceActivity) { + reportLogicalDeviceActivity(); + } + } + + private void reportLogicalDeviceActivity() { + systemContext.getDeviceStateService().onDeviceActivity(deviceId); } private void reportSessionOpen() { @@ -297,8 +326,67 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { return new HashSet<>(strings); } - private SessionType getSessionType(UUID sessionId) { - return sessions.containsKey(sessionId) ? SessionType.ASYNC : SessionType.SYNC; + private void handlePostAttributesRequest(ActorContext context, SessionInfoProto sessionInfo, PostAttributeMsg postAttributes) { + JsonObject json = JsonUtils.getJsonObject(postAttributes.getKvList()); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, defaultMetaData.copy(), + TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); + pushToRuleEngine(context, tbMsg); + } + + private void handlePostTelemetryRequest(ActorContext context, SessionInfoProto sessionInfo, PostTelemetryMsg postTelemetry) { + for (TsKvListProto tsKv : postTelemetry.getTsKvListList()) { + JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); + TbMsgMetaData metaData = defaultMetaData.copy(); + metaData.putValue("ts", tsKv.getTs() + ""); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); + pushToRuleEngine(context, tbMsg); + } + } + + private void handleClientSideRPCRequest(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg request) { + UUID sessionId = getSessionId(sessionInfo); + JsonObject json = new JsonObject(); + json.addProperty("method", request.getMethodName()); + json.add("params", JsonUtils.parse(request.getParams())); + + TbMsgMetaData requestMetaData = defaultMetaData.copy(); + requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); + context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); + + scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); + toServerRpcPendingMap.put(request.getRequestId(), new ToServerRpcRequestMetadata(sessionId, getSessionType(sessionId), sessionInfo.getNodeId())); + } + + private TransportProtos.SessionType getSessionType(UUID sessionId) { + return sessions.containsKey(sessionId) ? TransportProtos.SessionType.ASYNC : TransportProtos.SessionType.SYNC; + } + + void processClientSideRpcTimeout(ActorContext context, DeviceActorClientSideRpcTimeoutMsg msg) { + ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(msg.getId()); + if (data != null) { + log.debug("[{}] Client side RPC request [{}] timeout detected!", deviceId, msg.getId()); + sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(msg.getId()).setError("timeout").build() + , data.getSessionId(), data.getNodeId()); + } + } + + void processToServerRPCResponse(ActorContext context, ToServerRpcResponseActorMsg msg) { + int requestId = msg.getMsg().getRequestId(); + ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); + if (data != null) { + log.debug("[{}] Pushing reply to [{}][{}]!", deviceId, data.getNodeId(), data.getSessionId()); + sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(requestId).setPayload(msg.getMsg().getData()).build() + , data.getSessionId(), data.getNodeId()); + } else { + log.debug("[{}][{}] Pending RPC request to server not found!", deviceId, requestId); + } + } + + private void pushToRuleEngine(ActorContext context, TbMsg tbMsg) { + context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); } void processAttributesUpdate(ActorContext context, DeviceAttributesEventNotificationMsg msg) { @@ -346,7 +434,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(responseMsg.getRequestId()); boolean success = requestMd != null; if (success) { - systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), responseMsg.getPayload(), null)); } else { log.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId()); @@ -361,7 +449,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToAttributes(true); log.debug("[{}] Registering attributes subscription for session [{}]", deviceId, sessionId); @@ -382,7 +470,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToRPC(true); log.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId); @@ -406,11 +494,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { notifyTransportAboutClosedSession(sessionIdToRemove, sessions.remove(sessionIdToRemove)); } } - sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfo.getNodeId()))); + sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfo.getNodeId()))); if (sessions.size() == 1) { reportSessionOpen(); } - systemContext.getDeviceStateService().onDeviceActivity(deviceId, System.currentTimeMillis()); dumpSessions(); } else if (msg.getEvent() == SessionEvent.CLOSED) { log.debug("[{}] Canceling subscriptions for closed session [{}]", deviceId, sessionId); @@ -424,10 +511,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, SubscriptionInfoProto subscriptionInfo) { + private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, TransportProtos.SubscriptionInfoProto subscriptionInfo) { UUID sessionId = getSessionId(sessionInfoProto); SessionInfoMetaData sessionMD = sessions.computeIfAbsent(sessionId, - id -> new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); + id -> new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); sessionMD.setLastActivityTime(subscriptionInfo.getLastActivityTime()); sessionMD.setSubscribedToAttributes(subscriptionInfo.getAttributeSubscription()); @@ -438,7 +525,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (subscriptionInfo.getRpcSubscription()) { rpcSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo()); } - systemContext.getDeviceStateService().onDeviceActivity(deviceId, subscriptionInfo.getLastActivityTime()); dumpSessions(); } @@ -450,11 +536,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void notifyTransportAboutClosedSession(UUID sessionId, SessionInfoMetaData sessionMd) { - ToTransportMsg msg = ToTransportMsg.newBuilder() + DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setSessionCloseNotification(SessionCloseNotificationProto.getDefaultInstance()).build(); - systemContext.getTbCoreToTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); + systemContext.getRuleEngineTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); } void processNameOrTypeUpdate(DeviceNameOrTypeUpdateMsg msg) { @@ -466,35 +552,35 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendToTransport(GetAttributeResponseMsg responseMsg, SessionInfoProto sessionInfo) { - ToTransportMsg msg = ToTransportMsg.newBuilder() + DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() .setSessionIdMSB(sessionInfo.getSessionIdMSB()) .setSessionIdLSB(sessionInfo.getSessionIdLSB()) .setGetAttributesResponse(responseMsg).build(); - systemContext.getTbCoreToTransportService().process(sessionInfo.getNodeId(), msg); + systemContext.getRuleEngineTransportService().process(sessionInfo.getNodeId(), msg); } private void sendToTransport(AttributeUpdateNotificationMsg notificationMsg, UUID sessionId, String nodeId) { - ToTransportMsg msg = ToTransportMsg.newBuilder() + DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setAttributeUpdateNotification(notificationMsg).build(); - systemContext.getTbCoreToTransportService().process(nodeId, msg); + systemContext.getRuleEngineTransportService().process(nodeId, msg); } private void sendToTransport(ToDeviceRpcRequestMsg rpcMsg, UUID sessionId, String nodeId) { - ToTransportMsg msg = ToTransportMsg.newBuilder() + DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToDeviceRequest(rpcMsg).build(); - systemContext.getTbCoreToTransportService().process(nodeId, msg); + systemContext.getRuleEngineTransportService().process(nodeId, msg); } - private void sendToTransport(ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { - ToTransportMsg msg = ToTransportMsg.newBuilder() + private void sendToTransport(TransportProtos.ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { + DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToServerResponse(rpcMsg).build(); - systemContext.getTbCoreToTransportService().process(nodeId, msg); + systemContext.getRuleEngineTransportService().process(nodeId, msg); } @@ -546,9 +632,9 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void restoreSessions() { log.debug("[{}] Restoring sessions from cache", deviceId); - DeviceSessionsCacheEntry sessionsDump = null; + TransportProtos.DeviceSessionsCacheEntry sessionsDump = null; try { - sessionsDump = DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); + sessionsDump = TransportProtos.DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); } catch (InvalidProtocolBufferException e) { log.warn("[{}] Failed to decode device sessions from cache", deviceId); return; @@ -557,11 +643,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No session information found", deviceId); return; } - for (SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { + for (TransportProtos.SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { SessionInfoProto sessionInfoProto = sessionSubscriptionInfoProto.getSessionInfo(); UUID sessionId = getSessionId(sessionInfoProto); - SessionInfo sessionInfo = new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()); - SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); + SessionInfo sessionInfo = new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()); + TransportProtos.SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); SessionInfoMetaData sessionMD = new SessionInfoMetaData(sessionInfo, subInfo.getLastActivityTime()); sessions.put(sessionId, sessionMD); if (subInfo.getAttributeSubscription()) { @@ -579,27 +665,27 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void dumpSessions() { log.debug("[{}] Dumping sessions: {}, rpc subscriptions: {}, attribute subscriptions: {} to cache", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size()); - List sessionsList = new ArrayList<>(sessions.size()); + List sessionsList = new ArrayList<>(sessions.size()); sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getSessionInfo().getType() == SessionType.SYNC) { + if (sessionMD.getSessionInfo().getType() == TransportProtos.SessionType.SYNC) { return; } SessionInfo sessionInfo = sessionMD.getSessionInfo(); - SubscriptionInfoProto subscriptionInfoProto = SubscriptionInfoProto.newBuilder() + TransportProtos.SubscriptionInfoProto subscriptionInfoProto = TransportProtos.SubscriptionInfoProto.newBuilder() .setLastActivityTime(sessionMD.getLastActivityTime()) .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) .setRpcSubscription(sessionMD.isSubscribedToRPC()).build(); - SessionInfoProto sessionInfoProto = SessionInfoProto.newBuilder() + TransportProtos.SessionInfoProto sessionInfoProto = TransportProtos.SessionInfoProto.newBuilder() .setSessionIdMSB(uuid.getMostSignificantBits()) .setSessionIdLSB(uuid.getLeastSignificantBits()) .setNodeId(sessionInfo.getNodeId()).build(); - sessionsList.add(SessionSubscriptionInfoProto.newBuilder() + sessionsList.add(TransportProtos.SessionSubscriptionInfoProto.newBuilder() .setSessionInfo(sessionInfoProto) .setSubscriptionInfo(subscriptionInfoProto).build()); log.debug("[{}] Dumping session: {}", deviceId, sessionMD); }); systemContext.getDeviceSessionCacheService() - .put(deviceId, DeviceSessionsCacheEntry.newBuilder() + .put(deviceId, TransportProtos.DeviceSessionsCacheEntry.newBuilder() .addAllSessions(sessionsList).build().toByteArray()); } @@ -620,5 +706,4 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { dumpSessions(); } } - } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java similarity index 57% rename from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java rename to application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java index eb5b8e29d9..f49b2bdae8 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java @@ -13,20 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.rule.engine.flow; +package org.thingsboard.server.actors.device; +import akka.actor.ActorRef; import lombok.Data; -import org.thingsboard.rule.engine.api.NodeConfiguration; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.TbMsg; +/** + * Created by ashvayka on 15.03.18. + */ @Data -public class TbCheckpointNodeConfiguration implements NodeConfiguration { +public final class DeviceActorToRuleEngineMsg implements TbActorMsg { - private String queueName; + private final ActorRef callbackRef; + private final TbMsg tbMsg; @Override - public TbCheckpointNodeConfiguration defaultConfiguration() { - TbCheckpointNodeConfiguration configuration = new TbCheckpointNodeConfiguration(); - configuration.setQueueName("HighPriority"); - return configuration; + public MsgType getMsgType() { + return MsgType.DEVICE_ACTOR_TO_RULE_ENGINE_MSG; } } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java b/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java new file mode 100644 index 0000000000..8b502b973c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import akka.actor.ActorRef; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.rpc.GrpcSession; +import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; +import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class BasicRpcSessionListener implements GrpcSessionListener { + + private final ClusterRpcCallbackExecutorService callbackExecutorService; + private final ActorService service; + private final ActorRef manager; + private final ActorRef self; + + BasicRpcSessionListener(ActorSystemContext context, ActorRef manager, ActorRef self) { + this.service = context.getActorService(); + this.callbackExecutorService = context.getClusterRpcCallbackExecutor(); + this.manager = manager; + this.self = self; + } + + @Override + public void onConnected(GrpcSession session) { + log.info("[{}][{}] session started", session.getRemoteServer(), getType(session)); + if (!session.isClient()) { + manager.tell(new RpcSessionConnectedMsg(session.getRemoteServer(), session.getSessionId()), self); + } + } + + @Override + public void onDisconnected(GrpcSession session) { + log.info("[{}][{}] session closed", session.getRemoteServer(), getType(session)); + manager.tell(new RpcSessionDisconnectedMsg(session.isClient(), session.getRemoteServer()), self); + } + + @Override + public void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage) { + log.trace("Received session actor msg from [{}][{}]: {}", session.getRemoteServer(), getType(session), clusterMessage); + callbackExecutorService.execute(() -> { + try { + service.onReceivedMsg(session.getRemoteServer(), clusterMessage); + } catch (Exception e) { + log.debug("[{}][{}] Failed to process cluster message: {}", session.getRemoteServer(), getType(session), clusterMessage, e); + } + }); + } + + @Override + public void onError(GrpcSession session, Throwable t) { + log.warn("[{}][{}] session got error -> {}", session.getRemoteServer(), getType(session), t); + manager.tell(new RpcSessionClosedMsg(session.isClient(), session.getRemoteServer()), self); + session.close(); + } + + private static String getType(GrpcSession session) { + return session.isClient() ? "Client" : "Server"; + } + + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java similarity index 73% rename from common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java rename to application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java index e39dc0c322..f0ff8ce9e1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java @@ -13,14 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.settings; +package org.thingsboard.server.actors.rpc; import lombok.Data; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +/** + * @author Andrew Shvayka + */ @Data -public class TbRuleEngineQueueSubmitStrategyConfiguration { - - private String type; - private int batchSize; - +public final class RpcBroadcastMsg { + private final ClusterAPIProtos.ClusterMessage msg; } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java new file mode 100644 index 0000000000..133e7375d4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java @@ -0,0 +1,230 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import akka.actor.ActorRef; +import akka.actor.OneForOneStrategy; +import akka.actor.Props; +import akka.actor.SupervisorStrategy; +import akka.event.Logging; +import akka.event.LoggingAdapter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.discovery.ServerInstance; +import scala.concurrent.duration.Duration; + +import java.util.*; + +/** + * @author Andrew Shvayka + */ +public class RpcManagerActor extends ContextAwareActor { + + private final Map sessionActors; + private final Map> pendingMsgs; + private final ServerAddress instance; + + private RpcManagerActor(ActorSystemContext systemContext) { + super(systemContext); + this.sessionActors = new HashMap<>(); + this.pendingMsgs = new HashMap<>(); + this.instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); + + systemContext.getDiscoveryService().getOtherServers().stream() + .filter(otherServer -> otherServer.getServerAddress().compareTo(instance) > 0) + .forEach(otherServer -> onCreateSessionRequest( + new RpcSessionCreateRequestMsg(UUID.randomUUID(), otherServer.getServerAddress(), null))); + } + + @Override + protected boolean process(TbActorMsg msg) { + //TODO Move everything here, to work with TbActorMsg + return false; + } + + @Override + public void onReceive(Object msg) { + if (msg instanceof ClusterAPIProtos.ClusterMessage) { + onMsg((ClusterAPIProtos.ClusterMessage) msg); + } else if (msg instanceof RpcBroadcastMsg) { + onMsg((RpcBroadcastMsg) msg); + } else if (msg instanceof RpcSessionCreateRequestMsg) { + onCreateSessionRequest((RpcSessionCreateRequestMsg) msg); + } else if (msg instanceof RpcSessionConnectedMsg) { + onSessionConnected((RpcSessionConnectedMsg) msg); + } else if (msg instanceof RpcSessionDisconnectedMsg) { + onSessionDisconnected((RpcSessionDisconnectedMsg) msg); + } else if (msg instanceof RpcSessionClosedMsg) { + onSessionClosed((RpcSessionClosedMsg) msg); + } else if (msg instanceof ClusterEventMsg) { + onClusterEvent((ClusterEventMsg) msg); + } + } + + private void onMsg(RpcBroadcastMsg msg) { + log.debug("Forwarding msg to session actors {}", msg); + sessionActors.keySet().forEach(address -> { + ClusterAPIProtos.ClusterMessage msgWithServerAddress = msg.getMsg() + .toBuilder() + .setServerAddress(ClusterAPIProtos.ServerAddress + .newBuilder() + .setHost(address.getHost()) + .setPort(address.getPort()) + .build()) + .build(); + onMsg(msgWithServerAddress); + }); + pendingMsgs.values().forEach(queue -> queue.add(msg.getMsg())); + } + + private void onMsg(ClusterAPIProtos.ClusterMessage msg) { + if (msg.hasServerAddress()) { + ServerAddress address = new ServerAddress(msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE); + SessionActorInfo session = sessionActors.get(address); + if (session != null) { + log.debug("{} Forwarding msg to session actor: {}", address, msg); + session.getActor().tell(msg, ActorRef.noSender()); + } else { + log.debug("{} Storing msg to pending queue: {}", address, msg); + Queue queue = pendingMsgs.get(address); + if (queue == null) { + queue = new LinkedList<>(); + pendingMsgs.put(new ServerAddress( + msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE), queue); + } + queue.add(msg); + } + } else { + log.warn("Cluster msg doesn't have server address [{}]", msg); + } + } + + @Override + public void postStop() { + sessionActors.clear(); + pendingMsgs.clear(); + } + + private void onClusterEvent(ClusterEventMsg msg) { + ServerAddress server = msg.getServerAddress(); + if (server.compareTo(instance) > 0) { + if (msg.isAdded()) { + onCreateSessionRequest(new RpcSessionCreateRequestMsg(UUID.randomUUID(), server, null)); + } else { + onSessionClose(false, server); + } + } + } + + private void onSessionConnected(RpcSessionConnectedMsg msg) { + register(msg.getRemoteAddress(), msg.getId(), context().sender()); + } + + private void onSessionDisconnected(RpcSessionDisconnectedMsg msg) { + boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); + onSessionClose(reconnect, msg.getRemoteAddress()); + } + + private void onSessionClosed(RpcSessionClosedMsg msg) { + boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); + onSessionClose(reconnect, msg.getRemoteAddress()); + } + + private boolean isRegistered(ServerAddress address) { + for (ServerInstance server : systemContext.getDiscoveryService().getOtherServers()) { + if (server.getServerAddress().equals(address)) { + return true; + } + } + return false; + } + + private void onSessionClose(boolean reconnect, ServerAddress remoteAddress) { + log.info("[{}] session closed. Should reconnect: {}", remoteAddress, reconnect); + SessionActorInfo sessionRef = sessionActors.get(remoteAddress); + if (sessionRef != null && context().sender() != null && context().sender().equals(sessionRef.actor)) { + context().stop(sessionRef.actor); + sessionActors.remove(remoteAddress); + pendingMsgs.remove(remoteAddress); + if (reconnect) { + onCreateSessionRequest(new RpcSessionCreateRequestMsg(sessionRef.sessionId, remoteAddress, null)); + } + } + } + + private void onCreateSessionRequest(RpcSessionCreateRequestMsg msg) { + if (msg.getRemoteAddress() != null) { + if (!sessionActors.containsKey(msg.getRemoteAddress())) { + ActorRef actorRef = createSessionActor(msg); + register(msg.getRemoteAddress(), msg.getMsgUid(), actorRef); + } + } else { + createSessionActor(msg); + } + } + + private void register(ServerAddress remoteAddress, UUID uuid, ActorRef sender) { + sessionActors.put(remoteAddress, new SessionActorInfo(uuid, sender)); + log.info("[{}][{}] Registering session actor.", remoteAddress, uuid); + Queue data = pendingMsgs.remove(remoteAddress); + if (data != null) { + log.info("[{}][{}] Forwarding {} pending messages.", remoteAddress, uuid, data.size()); + data.forEach(msg -> sender.tell(new RpcSessionTellMsg(msg), ActorRef.noSender())); + } else { + log.info("[{}][{}] No pending messages to forward.", remoteAddress, uuid); + } + } + + private ActorRef createSessionActor(RpcSessionCreateRequestMsg msg) { + log.info("[{}] Creating session actor.", msg.getMsgUid()); + ActorRef actor = context().actorOf( + Props.create(new RpcSessionActor.ActorCreator(systemContext, msg.getMsgUid())) + .withDispatcher(DefaultActorService.RPC_DISPATCHER_NAME)); + actor.tell(msg, context().self()); + return actor; + } + + public static class ActorCreator extends ContextBasedCreator { + private static final long serialVersionUID = 1L; + + public ActorCreator(ActorSystemContext context) { + super(context); + } + + @Override + public RpcManagerActor create() { + return new RpcManagerActor(context); + } + } + + @Override + public SupervisorStrategy supervisorStrategy() { + return strategy; + } + + private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { + log.warn("Unknown failure", t); + return SupervisorStrategy.resume(); + }); +} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java new file mode 100644 index 0000000000..af2a7f0631 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java @@ -0,0 +1,135 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; +import org.thingsboard.server.service.cluster.rpc.GrpcSession; +import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; + +import java.util.UUID; + +import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE; + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class RpcSessionActor extends ContextAwareActor { + + + private final UUID sessionId; + private GrpcSession session; + private GrpcSessionListener listener; + + private RpcSessionActor(ActorSystemContext systemContext, UUID sessionId) { + super(systemContext); + this.sessionId = sessionId; + } + + @Override + protected boolean process(TbActorMsg msg) { + //TODO Move everything here, to work with TbActorMsg + return false; + } + + @Override + public void onReceive(Object msg) { + if (msg instanceof ClusterAPIProtos.ClusterMessage) { + tell((ClusterAPIProtos.ClusterMessage) msg); + } else if (msg instanceof RpcSessionCreateRequestMsg) { + initSession((RpcSessionCreateRequestMsg) msg); + } + } + + private void tell(ClusterAPIProtos.ClusterMessage msg) { + if (session != null) { + session.sendMsg(msg); + } else { + log.trace("Failed to send message due to missing session!"); + } + } + + @Override + public void postStop() { + if (session != null) { + log.info("Closing session -> {}", session.getRemoteServer()); + try { + session.close(); + } catch (RuntimeException e) { + log.trace("Failed to close session!", e); + } + } + } + + private void initSession(RpcSessionCreateRequestMsg msg) { + log.info("[{}] Initializing session", context().self()); + ServerAddress remoteServer = msg.getRemoteAddress(); + listener = new BasicRpcSessionListener(systemContext, context().parent(), context().self()); + if (msg.getRemoteAddress() == null) { + // Server session + session = new GrpcSession(listener); + session.setOutputStream(msg.getResponseObserver()); + session.initInputStream(); + session.initOutputStream(); + systemContext.getRpcService().onSessionCreated(msg.getMsgUid(), session.getInputStream()); + } else { + // Client session + ManagedChannel channel = ManagedChannelBuilder.forAddress(remoteServer.getHost(), remoteServer.getPort()).usePlaintext().build(); + session = new GrpcSession(remoteServer, listener, channel); + session.initInputStream(); + + ClusterRpcServiceGrpc.ClusterRpcServiceStub stub = ClusterRpcServiceGrpc.newStub(channel); + StreamObserver outputStream = stub.handleMsgs(session.getInputStream()); + + session.setOutputStream(outputStream); + session.initOutputStream(); + outputStream.onNext(toConnectMsg()); + } + } + + public static class ActorCreator extends ContextBasedCreator { + private static final long serialVersionUID = 1L; + + private final UUID sessionId; + + public ActorCreator(ActorSystemContext context, UUID sessionId) { + super(context); + this.sessionId = sessionId; + } + + @Override + public RpcSessionActor create() { + return new RpcSessionActor(context, sessionId); + } + } + + private ClusterAPIProtos.ClusterMessage toConnectMsg() { + ServerAddress instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); + return ClusterAPIProtos.ClusterMessage.newBuilder().setMessageType(CONNECT_RPC_MESSAGE).setServerAddress( + ClusterAPIProtos.ServerAddress.newBuilder().setHost(instance.getHost()) + .setPort(instance.getPort()).build()).build(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java new file mode 100644 index 0000000000..bb87d2ea88 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import lombok.Data; +import org.thingsboard.server.common.msg.cluster.ServerAddress; + +/** + * @author Andrew Shvayka + */ +@Data +public final class RpcSessionClosedMsg { + + private final boolean client; + private final ServerAddress remoteAddress; +} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java new file mode 100644 index 0000000000..489a5bfa17 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import lombok.Data; +import org.thingsboard.server.common.msg.cluster.ServerAddress; + +import java.util.UUID; + +/** + * @author Andrew Shvayka + */ +@Data +public final class RpcSessionConnectedMsg { + + private final ServerAddress remoteAddress; + private final UUID id; +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java similarity index 60% rename from application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java rename to application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java index 4a4829f0c2..df41a7e95f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java @@ -13,19 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.processing; +package org.thingsboard.server.actors.rpc; +import io.grpc.stub.StreamObserver; import lombok.Data; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.UUID; -import java.util.concurrent.ConcurrentMap; +/** + * @author Andrew Shvayka + */ @Data -public class TbRuleEngineProcessingDecision { +public final class RpcSessionCreateRequestMsg { - private final boolean commit; - private final ConcurrentMap> reprocessMap; + private final UUID msgUid; + private final ServerAddress remoteAddress; + private final StreamObserver responseObserver; } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java new file mode 100644 index 0000000000..14e7504637 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import lombok.Data; +import org.thingsboard.server.common.msg.cluster.ServerAddress; + +/** + * @author Andrew Shvayka + */ +@Data +public final class RpcSessionDisconnectedMsg { + + private final boolean client; + private final ServerAddress remoteAddress; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java similarity index 73% rename from common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java rename to application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java index 0d21c59c9c..3832d6eb94 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java @@ -13,16 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.settings; +package org.thingsboard.server.actors.rpc; import lombok.Data; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +/** + * @author Andrew Shvayka + */ @Data -public class TbRuleEngineQueueAckStrategyConfiguration { - - private String type; - private int retries; - private double failurePercentage; - private long pauseBetweenRetries; - +public final class RpcSessionTellMsg { + private final ClusterAPIProtos.ClusterMessage msg; } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java b/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java new file mode 100644 index 0000000000..811713819a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.rpc; + +import akka.actor.ActorRef; +import lombok.Data; + +import java.util.UUID; + +/** + * @author Andrew Shvayka + */ +@Data +public final class SessionActorInfo { + protected final UUID sessionId; + protected final ActorRef actor; +} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 6a3994a36f..b700007498 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -17,13 +17,18 @@ package org.thingsboard.server.actors.ruleChain; import akka.actor.ActorRef; import com.datastax.driver.core.ResultSetFuture; +import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; -import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.util.StringUtils; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.RuleChainTransactionService; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -35,15 +40,19 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -58,13 +67,11 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; import java.util.Collections; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -72,7 +79,6 @@ import java.util.function.Consumer; /** * Created by ashvayka on 19.03.18. */ -@Slf4j class DefaultTbContext implements TbContext { public final static ObjectMapper mapper = new ObjectMapper(); @@ -85,11 +91,6 @@ class DefaultTbContext implements TbContext { this.nodeCtx = nodeCtx; } - @Override - public void tellSuccess(TbMsg msg) { - tellNext(msg, Collections.singleton(TbRelationTypes.SUCCESS), null); - } - @Override public void tellNext(TbMsg msg, String relationType) { tellNext(msg, Collections.singleton(relationType), null); @@ -100,11 +101,16 @@ class DefaultTbContext implements TbContext { tellNext(msg, relationTypes, null); } + @Override + public void tellNext(TbMsg msg, String relationType, Throwable th) { + tellNext(msg, Collections.singleton(relationType), th); + } + private void tellNext(TbMsg msg, Set relationTypes, Throwable th) { if (nodeCtx.getSelf().isDebugMode()) { relationTypes.forEach(relationType -> mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, relationType, th)); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg), nodeCtx.getSelfActor()); } @Override @@ -113,94 +119,10 @@ class DefaultTbContext implements TbContext { scheduleMsgWithDelay(new RuleNodeToSelfMsg(msg), delayMs, nodeCtx.getSelfActor()); } - @Override - public void enqueue(TbMsg tbMsg, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueue(tpi, tbMsg, onFailure, onSuccess); - } - - @Override - public void enqueue(TbMsg tbMsg, String queueName, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); - enqueue(tpi, tbMsg, onFailure, onSuccess); - } - - private void enqueue(TopicPartitionInfo tpi, TbMsg tbMsg, Consumer onFailure, Runnable onSuccess) { - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, new SimpleTbQueueCallback(onSuccess, onFailure)); - } - - @Override - public void enqueueForTellFailure(TbMsg tbMsg, String failureMessage) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(TbRelationTypes.FAILURE), failureMessage, null, null); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, String relationType) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null, null); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, null, null, null); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, String relationType, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); - } - - @Override - public void enqueueForTellNext(TbMsg tbMsg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); - } - - private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, String failureMessage, Runnable onSuccess, Consumer onFailure) { - RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId(); - RuleNodeId ruleNodeId = nodeCtx.getSelf().getId(); - tbMsg = TbMsg.newMsg(tbMsg, ruleChainId, ruleNodeId); - TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)) - .addAllRelationTypes(relationTypes); - if (failureMessage != null) { - msg.setFailureMessage(failureMessage); - } - mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg.build(), new SimpleTbQueueCallback(onSuccess, onFailure)); - } - - @Override - public void ack(TbMsg tbMsg) { - if (nodeCtx.getSelf().isDebugMode()) { - mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "ACK", null); - } - tbMsg.getCallback().onSuccess(); - } - @Override public boolean isLocalEntity(EntityId entityId) { - return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); + Optional address = mainCtx.getRoutingService().resolveById(entityId); + return !address.isPresent(); } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -212,48 +134,66 @@ class DefaultTbContext implements TbContext { if (nodeCtx.getSelf().isDebugMode()) { mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, TbRelationTypes.FAILURE, th); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), - msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), msg), nodeCtx.getSelfActor()); } + @Override public void updateSelf(RuleNode self) { nodeCtx.setSelf(self); } @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return TbMsg.newMsg(type, originator, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); + return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), mainCtx.getQueuePartitionId()); } @Override public TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return TbMsg.transformMsg(origMsg, type, originator, metaData, data); + return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), mainCtx.getQueuePartitionId()); + } + + @Override + public void sendTbMsgToRuleEngine(TbMsg msg) { + mainCtx.getActorService().onMsg(new SendToClusterMsg(msg.getOriginator(), new ServiceToRuleEngineMsg(getTenantId(), msg))); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { - return entityCreatedMsg(customer, customer.getId(), ruleNodeId); + try { + ObjectNode entityNode = mapper.valueToTree(customer); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, customer.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + } catch (JsonProcessingException | IllegalArgumentException e) { + throw new RuntimeException("Failed to process customer created msg: " + e); + } } public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { - return entityCreatedMsg(device, device.getId(), ruleNodeId); + try { + ObjectNode entityNode = mapper.valueToTree(device); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, device.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + } catch (JsonProcessingException | IllegalArgumentException e) { + throw new RuntimeException("Failed to process device created msg: " + e); + } } public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { - return entityCreatedMsg(asset, asset.getId(), ruleNodeId); + try { + ObjectNode entityNode = mapper.valueToTree(asset); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, asset.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + } catch (JsonProcessingException | IllegalArgumentException e) { + throw new RuntimeException("Failed to process asset created msg: " + e); + } } public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) { - return entityCreatedMsg(alarm, alarm.getId(), ruleNodeId); - } - - public TbMsg entityCreatedMsg(E entity, I id, RuleNodeId ruleNodeId) { try { - return TbMsg.newMsg(DataConstants.ENTITY_CREATED, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); + ObjectNode entityNode = mapper.valueToTree(alarm); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, alarm.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " created msg: " + e); + throw new RuntimeException("Failed to process alarm created msg: " + e); } } + @Override public RuleNodeId getSelfId() { return nodeCtx.getSelf().getId(); @@ -311,8 +251,8 @@ class DefaultTbContext implements TbContext { } @Override - public String getServiceId() { - return mainCtx.getServiceInfoProvider().getServiceId(); + public String getNodeId() { + return mainCtx.getNodeIdProvider().getNodeId(); } @Override @@ -380,6 +320,11 @@ class DefaultTbContext implements TbContext { return mainCtx.getEntityViewService(); } + @Override + public RuleChainTransactionService getRuleChainTransactionService() { + return mainCtx.getRuleChainTransactionService(); + } + @Override public EventLoopGroup getSharedEventLoop() { return mainCtx.getSharedEventLoopGroupService().getSharedEventLoopGroup(); @@ -396,7 +341,35 @@ class DefaultTbContext implements TbContext { @Override public RuleEngineRpcService getRpcService() { - return mainCtx.getTbRuleEngineDeviceRpcService(); + return new RuleEngineRpcService() { + @Override + public void sendRpcReply(DeviceId deviceId, int requestId, String body) { + mainCtx.getDeviceRpcService().sendReplyToRpcCallFromDevice(nodeCtx.getTenantId(), deviceId, requestId, body); + } + + @Override + public void sendRpcRequest(RuleEngineDeviceRpcRequest src, Consumer consumer) { + ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), nodeCtx.getTenantId(), src.getDeviceId(), + src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); + mainCtx.getDeviceRpcService().forwardServerSideRPCRequestToDeviceActor(request, response -> { + if (src.isRestApiCall()) { + ServerAddress requestOriginAddress; + if (!StringUtils.isEmpty(src.getOriginHost())) { + requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); + } else { + requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); + } + mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); + } + consumer.accept(RuleEngineDeviceRpcResponse.builder() + .deviceId(src.getDeviceId()) + .requestId(src.getRequestId()) + .error(response.getError()) + .response(response.getResponse()) + .build()); + }); + } + }; } @Override @@ -414,6 +387,10 @@ class DefaultTbContext implements TbContext { return mainCtx.getRedisTemplate(); } + @Override + public String getServerAddress() { + return mainCtx.getServerAddress(); + } private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) { TbMsgMetaData metaData = new TbMsgMetaData(); @@ -421,29 +398,4 @@ class DefaultTbContext implements TbContext { return metaData; } - private class SimpleTbQueueCallback implements TbQueueCallback { - private final Runnable onSuccess; - private final Consumer onFailure; - - public SimpleTbQueueCallback(Runnable onSuccess, Consumer onFailure) { - this.onSuccess = onSuccess; - this.onFailure = onFailure; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (onSuccess != null) { - onSuccess.run(); - } - } - - @Override - public void onFailure(Throwable t) { - if (onFailure != null) { - onFailure.accept(t); - } else { - log.debug("[{}] Failed to put item into queue", nodeCtx.getTenantId(), t); - } - } - } } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java new file mode 100644 index 0000000000..d062c5dc49 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.ruleChain; + +import lombok.Data; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; +import org.thingsboard.server.common.msg.aware.TenantAwareMsg; + +import java.io.Serializable; + +/** + * Created by ashvayka on 19.03.18. + */ +@Data +final class RemoteToRuleChainTellNextMsg extends RuleNodeToRuleChainTellNextMsg implements TenantAwareMsg, RuleChainAwareMsg { + + private static final long serialVersionUID = 2459605482321657447L; + private final TenantId tenantId; + private final RuleChainId ruleChainId; + + public RemoteToRuleChainTellNextMsg(RuleNodeToRuleChainTellNextMsg original, TenantId tenantId, RuleChainId ruleChainId) { + super(original.getOriginator(), original.getRelationTypes(), original.getMsg()); + this.tenantId = tenantId; + this.ruleChainId = ruleChainId; + } + + @Override + public MsgType getMsgType() { + return MsgType.REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 027f4aac9d..72faaef533 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -15,25 +15,25 @@ */ package org.thingsboard.server.actors.ruleChain; +import akka.actor.ActorInitializationException; import akka.actor.OneForOneStrategy; import akka.actor.SupervisorStrategy; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.service.ComponentActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import scala.concurrent.duration.Duration; public class RuleChainActor extends ComponentActor { - private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChain ruleChain) { - super(systemContext, tenantId, ruleChain.getId()); - setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChain, systemContext, + private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId) { + super(systemContext, tenantId, ruleChainId); + setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChainId, systemContext, context().parent(), context().self())); } @@ -43,17 +43,20 @@ public class RuleChainActor extends ComponentActor { + private static final long DEFAULT_CLUSTER_PARTITION = 0L; private final ActorRef parent; private final ActorRef self; private final Map nodeActors; private final Map> nodeRoutes; private final RuleChainService service; - private final TbClusterService clusterService; - private String ruleChainName; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; + private String ruleChainName; - RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext + RuleChainActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { - super(systemContext, tenantId, ruleChain.getId()); - this.ruleChainName = ruleChain.getName(); + super(systemContext, tenantId, ruleChainId); this.parent = parent; this.self = self; this.nodeActors = new HashMap<>(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.clusterService = systemContext.getClusterService(); + this.ruleChainName = ruleChainId.toString(); } @Override @@ -96,6 +92,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor ruleNodeList = service.getRuleChainNodes(tenantId, entityId); log.trace("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); // Creating and starting the actors; @@ -155,8 +152,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actorRef.tell(msg, self)); + public void onClusterEventMsg(ClusterEventMsg msg) { + } private ActorRef createRuleNodeActor(ActorContext context, RuleNode ruleNode) { @@ -195,123 +192,100 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor relationTypes, String failureMessage) { - try { - checkActive(); - EntityId entityId = msg.getOriginator(); - TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - List relations = nodeRoutes.get(originatorNodeId).stream() - .filter(r -> contains(relationTypes, r.getType())) - .collect(Collectors.toList()); - int relationsCount = relations.size(); - if (relationsCount == 0) { - log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); - if (relationTypes.contains(TbRelationTypes.FAILURE)) { - RuleNodeCtx ruleNodeCtx = nodeActors.get(originatorNodeId); - if (ruleNodeCtx != null) { - msg.getCallback().onFailure(new RuleNodeException(failureMessage, ruleChainName, ruleNodeCtx.getSelf())); - } else { - log.debug("[{}] Failure during message processing by Rule Node [{}]. Enable and see debug events for more info", entityId, originatorNodeId.getId()); - msg.getCallback().onFailure(new RuleEngineException("Failure during message processing by Rule Node [" + originatorNodeId.getId().toString() + "]")); - } - } else { - msg.getCallback().onSuccess(); - } - } else if (relationsCount == 1) { - for (RuleNodeRelation relation : relations) { - log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - pushToTarget(tpi, msg, relation.getOut(), relation.getType()); - } + void onRuleChainToRuleChainMsg(RuleChainToRuleChainMsg envelope) { + checkActive(); + if (envelope.isEnqueue()) { + if (firstNode != null) { + pushMsgToNode(firstNode, enrichWithRuleChainId(envelope.getMsg()), envelope.getFromRelationType()); + } + } else { + if (firstNode != null) { + pushMsgToNode(firstNode, envelope.getMsg(), envelope.getFromRelationType()); } else { - MultipleTbQueueTbMsgCallbackWrapper callbackWrapper = new MultipleTbQueueTbMsgCallbackWrapper(relationsCount, msg.getCallback()); - log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relations); - for (RuleNodeRelation relation : relations) { - EntityId target = relation.getOut(); - putToQueue(tpi, msg, callbackWrapper, target); - } +// TODO: Ack this message in Kafka +// TbMsg msg = envelope.getMsg(); +// EntityId ackId = msg.getRuleNodeId() != null ? msg.getRuleNodeId() : msg.getRuleChainId(); +// queue.ack(tenantId, envelope.getMsg(), ackId.getId(), msg.getClusterPartition()); } - } catch (Exception e) { - msg.getCallback().onFailure(new RuleEngineException("onTellNext - " + e.getMessage())); } } - private void putToQueue(TopicPartitionInfo tpi, TbMsg msg, TbQueueCallback callbackWrapper, EntityId target) { - switch (target.getEntityType()) { - case RULE_NODE: - putToQueue(tpi, msg.copyWithRuleNodeId(entityId, new RuleNodeId(target.getId())), callbackWrapper); - break; - case RULE_CHAIN: - putToQueue(tpi, msg.copyWithRuleChainId(new RuleChainId(target.getId())), callbackWrapper); - break; + void onTellNext(RuleNodeToRuleChainTellNextMsg envelope) { + checkActive(); + TbMsg msg = envelope.getMsg(); + EntityId originatorEntityId = msg.getOriginator(); + Optional address = systemContext.getRoutingService().resolveById(originatorEntityId); + + if (address.isPresent()) { + onRemoteTellNext(address.get(), envelope); + } else { + onLocalTellNext(envelope); } } - private void pushToTarget(TopicPartitionInfo tpi, TbMsg msg, EntityId target, String fromRelationType) { - if (tpi.isMyPartition()) { - switch (target.getEntityType()) { - case RULE_NODE: - pushMsgToNode(nodeActors.get(new RuleNodeId(target.getId())), msg, fromRelationType); - break; - case RULE_CHAIN: - parent.tell(new RuleChainToRuleChainMsg(new RuleChainId(target.getId()), entityId, msg, fromRelationType), self); - break; + private void onRemoteTellNext(ServerAddress serverAddress, RuleNodeToRuleChainTellNextMsg envelope) { + TbMsg msg = envelope.getMsg(); + log.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator()); + envelope = new RemoteToRuleChainTellNextMsg(envelope, tenantId, entityId); + systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); + } + + private void onLocalTellNext(RuleNodeToRuleChainTellNextMsg envelope) { + TbMsg msg = envelope.getMsg(); + RuleNodeId originatorNodeId = envelope.getOriginator(); + List relations = nodeRoutes.get(originatorNodeId).stream() + .filter(r -> contains(envelope.getRelationTypes(), r.getType())) + .collect(Collectors.toList()); + int relationsCount = relations.size(); + EntityId ackId = msg.getRuleNodeId() != null ? msg.getRuleNodeId() : msg.getRuleChainId(); + if (relationsCount == 0) { + log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); + if (ackId != null) { +// TODO: Ack this message in Kafka +// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); + } + } else if (relationsCount == 1) { + for (RuleNodeRelation relation : relations) { + log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); + pushToTarget(msg, relation.getOut(), relation.getType()); } } else { - putToQueue(tpi, msg, new TbQueueTbMsgCallbackWrapper(msg.getCallback()), target); + for (RuleNodeRelation relation : relations) { + EntityId target = relation.getOut(); + log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); + switch (target.getEntityType()) { + case RULE_NODE: + enqueueAndForwardMsgCopyToNode(msg, target, relation.getType()); + break; + case RULE_CHAIN: + enqueueAndForwardMsgCopyToChain(msg, target, relation.getType()); + break; + } + } + //TODO: Ideally this should happen in async way when all targets confirm that the copied messages are successfully written to corresponding target queues. + if (ackId != null) { +// TODO: Ack this message in Kafka +// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); + } } } - private void putToQueue(TopicPartitionInfo tpi, TbMsg newMsg, TbQueueCallback callbackWrapper) { - ToRuleEngineMsg toQueueMsg = ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(newMsg)) - .build(); - clusterService.pushMsgToRuleEngine(tpi, newMsg.getId(), toQueueMsg, callbackWrapper); - } - private boolean contains(Set relationTypes, String type) { if (relationTypes == null) { return true; @@ -324,13 +298,38 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actors; - @Getter - protected RuleChain rootChain; - @Getter - protected ActorRef rootChainActor; + protected final RuleChainManager ruleChainManager; + protected final RuleChainService ruleChainService; - public RuleChainManagerActor(ActorSystemContext systemContext, TenantId tenantId) { + public RuleChainManagerActor(ActorSystemContext systemContext, RuleChainManager ruleChainManager) { super(systemContext); - this.tenantId = tenantId; - this.actors = HashBiMap.create(); + this.ruleChainManager = ruleChainManager; this.ruleChainService = systemContext.getRuleChainService(); } protected void initRuleChains() { - for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) { - RuleChainId ruleChainId = ruleChain.getId(); - log.debug("[{}|{}] Creating rule chain actor", ruleChainId.getEntityType(), ruleChain.getId()); - //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. - ActorRef actorRef = getOrCreateActor(this.context(), ruleChainId, id -> ruleChain); - visit(ruleChain, actorRef); - log.debug("[{}|{}] Rule Chain actor created.", ruleChainId.getEntityType(), ruleChainId.getId()); - } - } - - protected void visit(RuleChain entity, ActorRef actorRef) { - if (entity != null && entity.isRoot()) { - rootChain = entity; - rootChainActor = actorRef; - } - } - - public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId) { - return getOrCreateActor(context, ruleChainId, eId -> ruleChainService.findRuleChainById(TenantId.SYS_TENANT_ID, eId)); - } - - public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId, Function provider) { - return actors.computeIfAbsent(ruleChainId, eId -> { - RuleChain ruleChain = provider.apply(eId); - return context.actorOf(Props.create(new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain)) - .withDispatcher(DefaultActorService.TENANT_RULE_DISPATCHER_NAME), eId.toString()); - }); + ruleChainManager.init(this.context()); } protected ActorRef getEntityActorRef(EntityId entityId) { ActorRef target = null; - if (entityId.getEntityType() == EntityType.RULE_CHAIN) { - target = getOrCreateActor(this.context(), (RuleChainId) entityId); + switch (entityId.getEntityType()) { + case RULE_CHAIN: + target = ruleChainManager.getOrCreateActor(this.context(), (RuleChainId) entityId); + break; } return target; } protected void broadcast(Object msg) { - actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); + ruleChainManager.broadcast(msg); } - - public ActorRef get(RuleChainId id) { - return actors.get(id); - } - } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java index 0122471c52..e081768d93 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java @@ -32,6 +32,7 @@ public final class RuleChainToRuleChainMsg implements TbActorMsg, RuleChainAware private final RuleChainId source; private final TbMsg msg; private final String fromRelationType; + private final boolean enqueue; @Override public RuleChainId getRuleChainId() { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java index abb98468bf..65227818e8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; public class RuleNodeActor extends ComponentActor { @@ -54,9 +53,6 @@ public class RuleNodeActor extends ComponentActor relationTypes; private final TbMsg msg; - private final String failureMessage; @Override public MsgType getMsgType() { diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index f6403c5d56..3032b1dfe0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -15,7 +15,24 @@ */ package org.thingsboard.server.actors.service; -public interface ActorService { +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.transport.SessionMsgProcessor; +import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; +import org.thingsboard.server.service.cluster.rpc.RpcMsgListener; +public interface ActorService extends SessionMsgProcessor, RpcMsgListener, DiscoveryServiceListener { + + void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + + void onMsg(SendToClusterMsg msg); + + void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId); + + void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType); } diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java b/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java index 2a92caf4f6..3260c7ee31 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java @@ -22,7 +22,7 @@ import org.thingsboard.server.actors.stats.StatsPersistMsg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; /** @@ -115,9 +115,9 @@ public abstract class ComponentActor status = system.terminate(); @@ -97,4 +132,157 @@ public class DefaultActorService implements ActorService { } } + @Override + public void onMsg(SendToClusterMsg msg) { + appActor.tell(msg, ActorRef.noSender()); + } + + @Override + public void onServerAdded(ServerInstance server) { + log.trace("Processing onServerAdded msg: {}", server); + broadcast(new ClusterEventMsg(server.getServerAddress(), true)); + } + + @Override + public void onServerUpdated(ServerInstance server) { + //Do nothing + } + + @Override + public void onServerRemoved(ServerInstance server) { + log.trace("Processing onServerRemoved msg: {}", server); + broadcast(new ClusterEventMsg(server.getServerAddress(), false)); + } + + @Override + public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { + log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); + broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); + } + + @Override + public void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId) { + DeviceCredentialsUpdateNotificationMsg msg = new DeviceCredentialsUpdateNotificationMsg(tenantId, deviceId); + appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); + } + + @Override + public void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType) { + log.trace("[{}] Processing onDeviceNameOrTypeUpdate event, deviceName: {}, deviceType: {}", deviceId, deviceName, deviceType); + DeviceNameOrTypeUpdateMsg msg = new DeviceNameOrTypeUpdateMsg(tenantId, deviceId, deviceName, deviceType); + appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); + } + + public void broadcast(ToAllNodesMsg msg) { + actorContext.getEncodingService().encode(msg); + rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage + .newBuilder() + .setPayload(ByteString + .copyFrom(actorContext.getEncodingService().encode(msg))) + .setMessageType(CLUSTER_ACTOR_MESSAGE) + .build())); + appActor.tell(msg, ActorRef.noSender()); + } + + private void broadcast(ClusterEventMsg msg) { + this.appActor.tell(msg, ActorRef.noSender()); + this.rpcManagerActor.tell(msg, ActorRef.noSender()); + } + + @Value("${cluster.stats.enabled:false}") + private boolean statsEnabled; + + private final AtomicInteger sentClusterMsgs = new AtomicInteger(0); + private final AtomicInteger receivedClusterMsgs = new AtomicInteger(0); + + + @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + int sent = sentClusterMsgs.getAndSet(0); + int received = receivedClusterMsgs.getAndSet(0); + if (sent > 0 || received > 0) { + log.info("Cluster msgs sent [{}] received [{}]", sent, received); + } + } + } + + @Override + public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { + if (statsEnabled) { + receivedClusterMsgs.incrementAndGet(); + } + ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); + if (log.isDebugEnabled()) { + log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); + log.info("MSG: {}", msg); + } + switch (msg.getMessageType()) { + case CLUSTER_ACTOR_MESSAGE: + java.util.Optional decodedMsg = actorContext.getEncodingService() + .decode(msg.getPayload().toByteArray()); + if (decodedMsg.isPresent()) { + appActor.tell(decodedMsg.get(), ActorRef.noSender()); + } else { + log.error("Error during decoding cluster proto message"); + } + break; + case TO_ALL_NODES_MSG: + //TODO + break; + case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: + actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: + actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: + actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: + actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: + actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: + actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: + actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: + actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); + break; + case CLUSTER_TRANSACTION_SERVICE_MESSAGE: + actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); + break; + } + } + + @Override + public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { + if (statsEnabled) { + sentClusterMsgs.incrementAndGet(); + } + rpcManagerActor.tell(msg, ActorRef.noSender()); + } + + @Override + public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { + if (statsEnabled) { + sentClusterMsgs.incrementAndGet(); + } + rpcManagerActor.tell(msg, ActorRef.noSender()); + } + + @Override + public void onBroadcastMsg(RpcBroadcastMsg msg) { + rpcManagerActor.tell(msg, ActorRef.noSender()); + } + + @Override + public void onDeviceAdded(Device device) { + deviceStateService.onDeviceAdded(device); + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java index 1443c61e16..2fdc918f8e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java @@ -16,13 +16,20 @@ package org.thingsboard.server.actors.shared; import akka.actor.ActorContext; +import akka.event.LoggingAdapter; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.stats.StatsPersistTick; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; + +import javax.annotation.Nullable; +import java.util.function.Consumer; @Slf4j public abstract class ComponentMsgProcessor extends AbstractContextAwareMsgProcessor { @@ -43,7 +50,7 @@ public abstract class ComponentMsgProcessor extends Abstract public abstract void stop(ActorContext context) throws Exception; - public abstract void onPartitionChangeMsg(PartitionChangeMsg msg) throws Exception; + public abstract void onClusterEventMsg(ClusterEventMsg msg) throws Exception; public void onCreated(ActorContext context) throws Exception { start(context); diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java new file mode 100644 index 0000000000..b824cf61ff --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java @@ -0,0 +1,87 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.shared; + +import akka.actor.ActorContext; +import akka.actor.ActorRef; +import akka.actor.Props; +import akka.actor.UntypedActor; +import akka.japi.Creator; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.common.data.SearchTextBased; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.page.PageDataIterable; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by ashvayka on 15.03.18. + */ +@Slf4j +public abstract class EntityActorsManager> { + + protected final ActorSystemContext systemContext; + protected final BiMap actors; + + public EntityActorsManager(ActorSystemContext systemContext) { + this.systemContext = systemContext; + this.actors = HashBiMap.create(); + } + + protected abstract TenantId getTenantId(); + + protected abstract String getDispatcherName(); + + protected abstract Creator creator(T entityId); + + protected abstract PageDataIterable.FetchFunction getFetchEntitiesFunction(); + + public void init(ActorContext context) { + for (M entity : new PageDataIterable<>(getFetchEntitiesFunction(), ContextAwareActor.ENTITY_PACK_LIMIT)) { + T entityId = (T) entity.getId(); + log.debug("[{}|{}] Creating entity actor", entityId.getEntityType(), entityId.getId()); + //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. + ActorRef actorRef = getOrCreateActor(context, entityId); + visit(entity, actorRef); + log.debug("[{}|{}] Entity actor created.", entityId.getEntityType(), entityId.getId()); + } + } + + public void visit(M entity, ActorRef actorRef) { + } + + public ActorRef getOrCreateActor(ActorContext context, T entityId) { + return actors.computeIfAbsent(entityId, eId -> + context.actorOf(Props.create(creator(eId)) + .withDispatcher(getDispatcherName()), eId.toString())); + } + + public void broadcast(Object msg) { + actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); + } + + public void remove(T id) { + actors.remove(id); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java new file mode 100644 index 0000000000..ef2b4282d4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.shared.rulechain; + +import akka.actor.ActorRef; +import akka.japi.Creator; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.ruleChain.RuleChainActor; +import org.thingsboard.server.actors.shared.EntityActorsManager; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.dao.rule.RuleChainService; + +/** + * Created by ashvayka on 15.03.18. + */ +@Slf4j +public abstract class RuleChainManager extends EntityActorsManager { + + protected final RuleChainService service; + @Getter + protected RuleChain rootChain; + @Getter + protected ActorRef rootChainActor; + + public RuleChainManager(ActorSystemContext systemContext) { + super(systemContext); + this.service = systemContext.getRuleChainService(); + } + + @Override + public Creator creator(RuleChainId entityId) { + return new RuleChainActor.ActorCreator(systemContext, getTenantId(), entityId); + } + + @Override + public void visit(RuleChain entity, ActorRef actorRef) { + if (entity != null && entity.isRoot()) { + rootChain = entity; + rootChainActor = actorRef; + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java new file mode 100644 index 0000000000..10005b040b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.shared.rulechain; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.dao.model.ModelConstants; + +import java.util.Collections; + +public class SystemRuleChainManager extends RuleChainManager { + + public SystemRuleChainManager(ActorSystemContext systemContext) { + super(systemContext); + } + + @Override + protected FetchFunction getFetchEntitiesFunction() { + return link -> new PageData<>(); + } + + @Override + protected TenantId getTenantId() { + return ModelConstants.SYSTEM_TENANT; + } + + @Override + protected String getDispatcherName() { + return DefaultActorService.SYSTEM_RULE_DISPATCHER_NAME; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java new file mode 100644 index 0000000000..cfe862af49 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.shared.rulechain; + +import akka.actor.ActorContext; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; +import org.thingsboard.server.common.data.rule.RuleChain; + +public class TenantRuleChainManager extends RuleChainManager { + + private final TenantId tenantId; + + public TenantRuleChainManager(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.tenantId = tenantId; + } + + @Override + public void init(ActorContext context) { + super.init(context); + } + + @Override + protected TenantId getTenantId() { + return tenantId; + } + + @Override + protected String getDispatcherName() { + return DefaultActorService.TENANT_RULE_DISPATCHER_NAME; + } + + @Override + protected FetchFunction getFetchEntitiesFunction() { + return link -> service.findTenantRuleChains(tenantId, link); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 717dcecc19..31461a8fac 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -24,6 +24,7 @@ import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; @Slf4j public class StatsActor extends ContextAwareActor { @@ -57,12 +58,12 @@ public class StatsActor extends ContextAwareActor { event.setEntityId(msg.getEntityId()); event.setTenantId(msg.getTenantId()); event.setType(DataConstants.STATS); - event.setBody(toBodyJson(systemContext.getServiceInfoProvider().getServiceId(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); + event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); systemContext.getEventService().save(event); } - private JsonNode toBodyJson(String serviceId, long messagesProcessed, long errorsOccurred) { - return mapper.createObjectNode().put("server", serviceId).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); + private JsonNode toBodyJson(ServerAddress server, long messagesProcessed, long errorsOccurred) { + return mapper.createObjectNode().put("server", server.toString()).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); } public static class ActorCreator extends ContextBasedCreator { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 3cee82127c..4aacfb55bf 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -22,43 +22,41 @@ import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; import akka.actor.Terminated; +import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.device.DeviceActorCreator; +import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.actors.shared.rulechain.TenantRuleChainManager; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import scala.concurrent.duration.Duration; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; +import java.util.HashMap; +import java.util.Map; public class TenantActor extends RuleChainManagerActor { + private final TenantId tenantId; private final BiMap deviceActors; - private boolean isRuleEngineForCurrentTenant; - private boolean isCore; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext, tenantId); + super(systemContext, new TenantRuleChainManager(systemContext, tenantId)); + this.tenantId = tenantId; this.deviceActors = HashBiMap.create(); } @@ -67,37 +65,12 @@ public class TenantActor extends RuleChainManagerActor { return strategy; } - boolean cantFindTenant = false; - @Override public void preStart() { log.info("[{}] Starting tenant actor.", tenantId); try { - Tenant tenant = systemContext.getTenantService().findTenantById(tenantId); - if (tenant == null) { - cantFindTenant = true; - log.info("[{}] Started tenant actor for missing tenant.", tenantId); - } else { - // This Service may be started for specific tenant only. - Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); - - isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); - isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - - if (isRuleEngineForCurrentTenant) { - try { - if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { - log.info("[{}] Going to init rule chains", tenantId); - initRuleChains(); - } else { - isRuleEngineForCurrentTenant = false; - } - } catch (Exception e) { - cantFindTenant = true; - } - } - log.info("[{}] Tenant actor started.", tenantId); - } + initRuleChains(); + log.info("[{}] Tenant actor started.", tenantId); } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); } @@ -110,36 +83,18 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { - if (cantFindTenant) { - log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg); - if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { - QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; - queueMsg.getTbMsg().getCallback().onSuccess(); - } - return true; - } switch (msg.getMsgType()) { - case PARTITION_CHANGE_MSG: - PartitionChangeMsg partitionChangeMsg = (PartitionChangeMsg) msg; - ServiceType serviceType = partitionChangeMsg.getServiceQueueKey().getServiceType(); - if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { - //To Rule Chain Actors - broadcast(msg); - } else if (ServiceType.TB_CORE.equals(serviceType)) { - //To Device Actors - List repartitionedDevices = - deviceActors.keySet().stream().filter(deviceId -> !isMyPartition(deviceId)).collect(Collectors.toList()); - for (DeviceId deviceId : repartitionedDevices) { - ActorRef deviceActor = deviceActors.remove(deviceId); - context().stop(deviceActor); - } - } + case CLUSTER_EVENT_MSG: + broadcast(msg); break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case QUEUE_TO_RULE_ENGINE_MSG: - onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); + case SERVICE_TO_RULE_ENGINE_MSG: + onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); + break; + case DEVICE_ACTOR_TO_RULE_ENGINE_MSG: + onDeviceActorToRuleEngineMsg((DeviceActorToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -150,6 +105,7 @@ public class TenantActor extends RuleChainManagerActor { onToDeviceActorMsg((DeviceAwareMsg) msg); break; case RULE_CHAIN_TO_RULE_CHAIN_MSG: + case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; default: @@ -158,59 +114,41 @@ public class TenantActor extends RuleChainManagerActor { return true; } - private boolean isMyPartition(DeviceId deviceId) { - return systemContext.resolve(ServiceType.TB_CORE, tenantId, deviceId).isMyPartition(); + private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { + if (ruleChainManager.getRootChainActor() != null) { + ruleChainManager.getRootChainActor().tell(msg, self()); + } else { + log.info("[{}] No Root Chain: {}", tenantId, msg); + } } - private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { - if (!isRuleEngineForCurrentTenant) { - log.warn("RECEIVED INVALID MESSAGE: {}", msg); - return; - } - TbMsg tbMsg = msg.getTbMsg(); - if (tbMsg.getRuleChainId() == null) { - if (getRootChainActor() != null) { - getRootChainActor().tell(msg, self()); - } else { - tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); - log.info("[{}] No Root Chain: {}", tenantId, msg); - } + private void onDeviceActorToRuleEngineMsg(DeviceActorToRuleEngineMsg msg) { + if (ruleChainManager.getRootChainActor() != null) { + ruleChainManager.getRootChainActor().tell(msg, self()); } else { - ActorRef ruleChainActor = get(tbMsg.getRuleChainId()); - if (ruleChainActor != null) { - ruleChainActor.tell(msg, self()); - } else { - log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId()); - //TODO: 3.1 Log it to dead letters queue; - tbMsg.getCallback().onSuccess(); - } + log.info("[{}] No Root Chain: {}", tenantId, msg); } } private void onRuleChainMsg(RuleChainAwareMsg msg) { - getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); + ruleChainManager.getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); } private void onToDeviceActorMsg(DeviceAwareMsg msg) { - if (!isCore) { - log.warn("RECEIVED INVALID MESSAGE: {}", msg); - } getOrCreateDeviceActor(msg.getDeviceId()).tell(msg, ActorRef.noSender()); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { - if (isRuleEngineForCurrentTenant) { - ActorRef target = getEntityActorRef(msg.getEntityId()); - if (target != null) { - if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { - RuleChain ruleChain = systemContext.getRuleChainService(). - findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); - visit(ruleChain, target); - } - target.tell(msg, ActorRef.noSender()); - } else { - log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); + ActorRef target = getEntityActorRef(msg.getEntityId()); + if (target != null) { + if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { + RuleChain ruleChain = systemContext.getRuleChainService(). + findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); + ruleChainManager.visit(ruleChain, target); } + target.tell(msg, ActorRef.noSender()); + } else { + log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } } @@ -234,7 +172,7 @@ public class TenantActor extends RuleChainManagerActor { if (removed) { log.debug("[{}] Removed actor:", terminated); } else { - log.debug("Removed actor was not found in the device map!"); + log.warn("[{}] Removed actor was not found in the device map!"); } } else { throw new IllegalStateException("Remote actors are not supported!"); @@ -257,12 +195,15 @@ public class TenantActor extends RuleChainManagerActor { } } - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { - log.warn("[{}] Unknown failure", tenantId, t); - if (t instanceof ActorInitializationException) { - return SupervisorStrategy.stop(); - } else { - return SupervisorStrategy.resume(); + private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), new Function() { + @Override + public SupervisorStrategy.Directive apply(Throwable t) { + log.warn("[{}] Unknown failure", tenantId, t); + if (t instanceof ActorInitializationException) { + return SupervisorStrategy.stop(); + } else { + return SupervisorStrategy.resume(); + } } }); diff --git a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java index e57695da69..5e5bd94afa 100644 --- a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java @@ -32,13 +32,11 @@ import org.springframework.web.socket.server.support.HttpSessionHandshakeInterce import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.controller.plugin.TbWebSocketHandler; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; @Configuration -@TbCoreComponent @EnableWebSocket public class WebSocketConfiguration implements WebSocketConfigurer { @@ -64,8 +62,7 @@ public class WebSocketConfiguration implements WebSocketConfigurer { SecurityUser user = null; try { user = getCurrentUser(); - } catch (ThingsboardException ex) { - } + } catch (ThingsboardException ex) {} if (user == null) { response.setStatusCode(HttpStatus.UNAUTHORIZED); return false; diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 8873d82366..b7efc180a5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -25,25 +25,23 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.update.UpdateService; +import org.thingsboard.server.common.data.UpdateMessage; @RestController -@TbCoreComponent @RequestMapping("/api/admin") public class AdminController extends BaseController { @Autowired private MailService mailService; - + @Autowired private AdminSettingsService adminSettingsService; @@ -67,7 +65,7 @@ public class AdminController extends BaseController { @PreAuthorize("hasAuthority('SYS_ADMIN')") @RequestMapping(value = "/settings", method = RequestMethod.POST) - @ResponseBody + @ResponseBody public AdminSettings saveAdminSettings(@RequestBody AdminSettings adminSettings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); @@ -113,8 +111,8 @@ public class AdminController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); adminSettings = checkNotNull(adminSettings); if (adminSettings.getKey().equals("mail")) { - String email = getCurrentUser().getEmail(); - mailService.sendTestMail(adminSettings.getJsonValue(), email); + String email = getCurrentUser().getEmail(); + mailService.sendTestMail(adminSettings.getJsonValue(), email); } } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 85ee63b3b1..9999ab6709 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; @@ -41,14 +41,12 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.UUID; @RestController -@TbCoreComponent @RequestMapping("/api") public class AlarmController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 6963a35d6f..87a5ab3fc4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -33,15 +33,16 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -51,7 +52,6 @@ import java.util.List; import java.util.stream.Collectors; @RestController -@TbCoreComponent @RequestMapping("/api") public class AssetController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java index 7eb74fef90..8612d9b027 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.Arrays; import java.util.List; @@ -40,7 +39,6 @@ import java.util.UUID; import java.util.stream.Collectors; @RestController -@TbCoreComponent @RequestMapping("/api") public class AuditLogController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 42da043a91..44449eb3d6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -40,7 +40,6 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; import org.thingsboard.server.common.data.security.model.SecuritySettings; @@ -57,7 +56,6 @@ import java.net.URI; import java.net.URISyntaxException; @RestController -@TbCoreComponent @RequestMapping("/api") @Slf4j public class AuthController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 36bc2b104c..02e0cef53c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -26,27 +27,16 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.DashboardInfo; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceInfo; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.EntityViewInfo; -import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.User; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; @@ -74,6 +64,8 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -94,11 +86,7 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.component.ComponentDiscoveryService; -import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; @@ -117,7 +105,6 @@ import java.util.UUID; import static org.thingsboard.server.dao.service.Validator.validateId; @Slf4j -@TbCoreComponent public abstract class BaseController { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; @@ -168,7 +155,7 @@ public abstract class BaseController { protected RuleChainService ruleChainService; @Autowired - protected TbClusterService tbClusterService; + protected ActorService actorService; @Autowired protected RelationService relationService; @@ -191,12 +178,6 @@ public abstract class BaseController { @Autowired protected ClaimDevicesService claimDevicesService; - @Autowired - protected PartitionService partitionService; - - @Autowired - protected TbQueueProducerProvider producerProvider; - @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -713,14 +694,10 @@ public abstract class BaseController { } } } - TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); - TenantId tenantId = user.getTenantId(); - if (tenantId.isNullUid()) { - if (entity instanceof HasTenantId) { - tenantId = ((HasTenantId) entity).getTenantId(); - } - } - tbClusterService.pushMsgToRuleEngine(tenantId, entityId, tbMsg, null); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, entityId, metaData, TbMsgDataType.JSON + , json.writeValueAsString(entityNode) + , null, null, 0L); + actorService.onMsg(new SendToClusterMsg(entityId, new ServiceToRuleEngineMsg(user.getTenantId(), tbMsg))); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java index d75da8b34c..ef9508ad7c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java +++ b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java @@ -25,14 +25,12 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; import org.thingsboard.server.common.data.plugin.ComponentType; -import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.HashSet; import java.util.List; import java.util.Set; @RestController -@TbCoreComponent @RequestMapping("/api") public class ComponentDescriptorController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index 5d7e662a72..65c031172d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -36,12 +36,10 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController -@TbCoreComponent @RequestMapping("/api") public class CustomerController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index 1b972a5319..a0b3904d7a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; @@ -39,7 +40,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -47,7 +47,6 @@ import java.util.HashSet; import java.util.Set; @RestController -@TbCoreComponent @RequestMapping("/api") public class DashboardController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index abf00e2bca..c2e82f897e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -31,8 +31,6 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg; -import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.data.ClaimRequest; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; @@ -53,7 +51,6 @@ import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -65,7 +62,6 @@ import java.util.List; import java.util.stream.Collectors; @RestController -@TbCoreComponent @RequestMapping("/api") public class DeviceController extends BaseController { @@ -113,8 +109,12 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), - savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); + actorService + .onDeviceNameOrTypeUpdate( + savedDevice.getTenantId(), + savedDevice.getId(), + savedDevice.getName(), + savedDevice.getType()); logEntityAction(savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), @@ -267,9 +267,7 @@ public class DeviceController extends BaseController { try { Device device = checkDeviceId(deviceCredentials.getDeviceId(), Operation.WRITE_CREDENTIALS); DeviceCredentials result = checkNotNull(deviceCredentialsService.updateDeviceCredentials(getCurrentUser().getTenantId(), deviceCredentials)); - - tbClusterService.pushMsgToCore(new DeviceCredentialsUpdateNotificationMsg(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()), null); - + actorService.onCredentialsUpdate(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()); logEntityAction(device.getId(), device, device.getCustomerId(), ActionType.CREDENTIALS_UPDATED, null, deviceCredentials); diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java index 88aa9ce850..8058cca685 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java @@ -33,7 +33,6 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import java.util.List; @@ -41,7 +40,6 @@ import java.util.stream.Collectors; @RestController -@TbCoreComponent @RequestMapping("/api") public class EntityRelationController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java index 1611e71c27..67c8ea0688 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java @@ -44,7 +44,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -62,7 +61,6 @@ import static org.thingsboard.server.controller.CustomerController.CUSTOMER_ID; * Created by Victor Basanets on 8/28/2017. */ @RestController -@TbCoreComponent @RequestMapping("/api") @Slf4j public class EntityViewController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EventController.java b/application/src/main/java/org/thingsboard/server/controller/EventController.java index cf868b29a2..cba2b2bcc0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EventController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EventController.java @@ -31,11 +31,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; @RestController -@TbCoreComponent @RequestMapping("/api") public class EventController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java deleted file mode 100644 index a0b46f9166..0000000000 --- a/application/src/main/java/org/thingsboard/server/controller/QueueController.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.controller; - -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -@RestController -@TbCoreComponent -@RequestMapping("/api") -public class QueueController extends BaseController { - - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/tenant/queues", params = {"serviceType"}, method = RequestMethod.GET) - @ResponseBody - public List getTenantQueuesByServiceType(@RequestParam String serviceType) throws ThingsboardException { - checkParameter("serviceType", serviceType); - try { - ServiceType type = ServiceType.valueOf(serviceType); - switch (type) { - case TB_RULE_ENGINE: - return Arrays.asList("Main", "HighPriority", "SequentialByOriginator"); - default: - return Collections.emptyList(); - } - } catch (Exception e) { - throw handleException(e); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index b88c3f3695..a8298ce82b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -31,6 +31,7 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; @@ -42,25 +43,27 @@ import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.rpc.RpcRequest; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.rpc.DeviceRpcService; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.LocalRequestMetaData; -import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.telemetry.exception.ToErrorResponseEntity; import javax.annotation.Nullable; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.io.IOException; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; /** * Created by ashvayka on 22.03.18. */ @RestController -@TbCoreComponent @RequestMapping(TbUrlConstants.RPC_URL_PREFIX) @Slf4j public class RpcController extends BaseController { @@ -69,7 +72,7 @@ public class RpcController extends BaseController { protected final ObjectMapper jsonMapper = new ObjectMapper(); @Autowired - private TbCoreDeviceRpcService deviceRpcService; + private DeviceRpcService deviceRpcService; @Autowired private AccessValidator accessValidator; @@ -88,6 +91,7 @@ public class RpcController extends BaseController { return handleDeviceRPCRequest(false, new DeviceId(UUID.fromString(deviceIdStr)), requestBody); } + private DeferredResult handleDeviceRPCRequest(boolean oneWay, DeviceId deviceId, String requestBody) throws ThingsboardException { try { JsonNode rpcRequestBody = jsonMapper.readTree(requestBody); @@ -112,7 +116,7 @@ public class RpcController extends BaseController { timeout, body ); - deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); + deviceRpcService.processRestAPIRpcRequestToRuleEngine(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index a97b976bb7..cc6d4447e9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -52,10 +52,8 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import org.thingsboard.server.service.security.permission.Operation; @@ -69,7 +67,6 @@ import java.util.stream.Collectors; @Slf4j @RestController -@TbCoreComponent @RequestMapping("/api") public class RuleChainController extends BaseController { @@ -133,7 +130,7 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - tbClusterService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), + actorService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); logEntityAction(savedRuleChain.getId(), savedRuleChain, @@ -164,7 +161,7 @@ public class RuleChainController extends BaseController { previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), + actorService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, @@ -172,7 +169,7 @@ public class RuleChainController extends BaseController { ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId); - tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), + actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, @@ -206,7 +203,7 @@ public class RuleChainController extends BaseController { RuleChain ruleChain = checkRuleChain(ruleChainMetaData.getRuleChainId(), Operation.WRITE); RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData)); - tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); + actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, null, @@ -258,9 +255,9 @@ public class RuleChainController extends BaseController { referencingRuleChainIds.remove(ruleChain.getId()); referencingRuleChainIds.forEach(referencingRuleChainId -> - tbClusterService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); + actorService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); - tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); + actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); logEntityAction(ruleChainId, ruleChain, null, @@ -321,7 +318,7 @@ public class RuleChainController extends BaseController { ScriptEngine engine = null; try { engine = new RuleNodeJsScriptEngine(jsInvokeService, getCurrentUser().getId(), script, argNames); - TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); + TbMsg inMsg = new TbMsg(UUIDs.timeBased(), msgType, null, new TbMsgMetaData(metadata), data, null, null, 0L); switch (scriptType) { case "update": output = msgToOutput(engine.executeUpdate(inMsg)); diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index bb866e3e9b..f607d57cf6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -69,9 +69,9 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -99,7 +99,6 @@ import java.util.stream.Collectors; * Created by ashvayka on 22.03.18. */ @RestController -@TbCoreComponent @RequestMapping(TbUrlConstants.TELEMETRY_URL_PREFIX) @Slf4j public class TelemetryController extends BaseController { @@ -344,7 +343,7 @@ public class TelemetryController extends BaseController { return deleteAttributes(entityId, scope, keysStr); } - private DeferredResult deleteAttributes(EntityId entityIdSrc, String scope, String keysStr) throws ThingsboardException { + private DeferredResult deleteAttributes(EntityId entityIdStr, String scope, String keysStr) throws ThingsboardException { List keys = toKeysList(keysStr); if (keys.isEmpty()) { return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST); @@ -354,18 +353,19 @@ public class TelemetryController extends BaseController { if (DataConstants.SERVER_SCOPE.equals(scope) || DataConstants.SHARED_SCOPE.equals(scope) || DataConstants.CLIENT_SCOPE.equals(scope)) { - return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { + return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdStr, (result, tenantId, entityId) -> { ListenableFuture> future = attributesService.removeAll(user.getTenantId(), entityId, scope, keys); Futures.addCallback(future, new FutureCallback>() { @Override public void onSuccess(@Nullable List tmp) { logAttributesDeleted(user, entityId, scope, keys, null); - if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { + if (entityId.getEntityType() == EntityType.DEVICE) { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); - tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete( - user.getTenantId(), deviceId, keysToNotify), null); + DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onDelete( + user.getTenantId(), deviceId, keysToNotify); + actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); } result.setResult(new ResponseEntity<>(HttpStatus.OK)); } @@ -397,6 +397,12 @@ public class TelemetryController extends BaseController { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); + if (entityId.getEntityType() == EntityType.DEVICE) { + DeviceId deviceId = new DeviceId(entityId.getId()); + DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate( + user.getTenantId(), deviceId, scope, attributes); + actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + } result.setResult(new ResponseEntity(HttpStatus.OK)); } diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 18d3e80222..f9781269d7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -34,13 +34,11 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController -@TbCoreComponent @RequestMapping("/api") @Slf4j public class TenantController extends BaseController { @@ -76,6 +74,7 @@ public class TenantController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.TENANT, operation, tenant.getId(), tenant); + tenant = checkNotNull(tenantService.saveTenant(tenant)); if (newTenant) { installScripts.createDefaultRuleChains(tenant.getId()); @@ -95,7 +94,8 @@ public class TenantController extends BaseController { TenantId tenantId = new TenantId(toUUID(strTenantId)); checkTenantId(tenantId, Operation.DELETE); tenantService.deleteTenant(tenantId); - tbClusterService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); + + actorService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index 7f190f93cd..c93d672e07 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -44,7 +44,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -56,7 +55,6 @@ import org.thingsboard.server.service.security.permission.Resource; import javax.servlet.http.HttpServletRequest; @RestController -@TbCoreComponent @RequestMapping("/api") public class UserController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java index debe49b018..ce3e65c0a1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java @@ -31,14 +31,12 @@ import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController -@TbCoreComponent @RequestMapping("/api") public class WidgetTypeController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java index c0cf672c6f..b13b3aea54 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java @@ -32,14 +32,13 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController -@TbCoreComponent @RequestMapping("/api") public class WidgetsBundleController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index 6e33133522..f2862c5ab6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -33,7 +33,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.config.WebSocketConfiguration; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.telemetry.SessionEvent; @@ -53,7 +52,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; @Service -@TbCoreComponent @Slf4j public class TbWebSocketHandler extends TextWebSocketHandler implements TelemetryWebSocketMsgEndpoint { @@ -65,6 +63,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr @Value("${server.ws.send_timeout:5000}") private long sendTimeout; + @Value("${server.ws.limits.max_sessions_per_tenant:0}") private int maxSessionsPerTenant; @Value("${server.ws.limits.max_sessions_per_customer:0}") @@ -92,10 +91,10 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr try { SessionMetaData sessionMd = internalSessionMap.get(session.getId()); if (sessionMd != null) { - log.trace("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); + log.info("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); webSocketService.handleWebSocketMsg(sessionMd.sessionRef, message.getPayload()); } else { - log.trace("[{}] Failed to find session", session.getId()); + log.warn("[{}] Failed to find session", session.getId()); session.close(CloseStatus.SERVER_ERROR.withReason("Session not found!")); } } catch (IOException e) { @@ -139,7 +138,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr if (sessionMd != null) { processInWebSocketService(sessionMd.sessionRef, SessionEvent.onError(tError)); } else { - log.trace("[{}] Failed to find session", session.getId()); + log.warn("[{}] Failed to find session", session.getId()); } log.trace("[{}] Session transport error", session.getId(), tError); } diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 7137155ad5..e0daf43bf7 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -23,8 +23,8 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.service.component.ComponentDiscoveryService; -import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; import org.thingsboard.server.service.install.DatabaseTsUpgradeService; +import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.TsDatabaseSchemaService; @@ -152,11 +152,6 @@ public class ThingsboardInstallService { log.info("Updating system data..."); - systemDataLoaderService.updateSystemWidgets(); - break; - case "2.5.0": - log.info("Upgrading ThingsBoard from version 2.5 to 3.0 ..."); - log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java new file mode 100644 index 0000000000..9a7c83f167 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.discovery; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; + +import javax.annotation.PostConstruct; + +import static org.thingsboard.server.utils.MiscUtils.missingProperty; + +/** + * @author Andrew Shvayka + */ +@Service +@Slf4j +public class CurrentServerInstanceService implements ServerInstanceService { + + @Value("${rpc.bind_host}") + private String rpcHost; + @Value("${rpc.bind_port}") + private Integer rpcPort; + + private ServerInstance self; + + @PostConstruct + public void init() { + Assert.hasLength(rpcHost, missingProperty("rpc.bind_host")); + Assert.notNull(rpcPort, missingProperty("rpc.bind_port")); + self = new ServerInstance(new ServerAddress(rpcHost, rpcPort, ServerType.CORE)); + log.info("Current server instance: [{};{}]", self.getHost(), self.getPort()); + } + + @Override + public ServerInstance getSelf() { + return self; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java similarity index 70% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java rename to application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java index 36510261d6..d0ba27814c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java @@ -13,8 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.service.cluster.discovery; +import java.util.List; + +/** + * @author Andrew Shvayka + */ public interface DiscoveryService { + void publishCurrentServer(); + + void unpublishCurrentServer(); + + ServerInstance getCurrentServer(); + + List getOtherServers(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java new file mode 100644 index 0000000000..ef6b565817 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.discovery; + +/** + * @author Andrew Shvayka + */ +public interface DiscoveryServiceListener { + + void onServerAdded(ServerInstance server); + + void onServerUpdated(ServerInstance server); + + void onServerRemoved(ServerInstance server); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java similarity index 58% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java rename to application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java index 9017823efb..009a7801e7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java @@ -13,35 +13,55 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.service.cluster.discovery; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.annotation.DependsOn; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; import java.util.Collections; +import java.util.List; +/** + * @author Andrew Shvayka + */ @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) @Slf4j @DependsOn("environmentLogService") public class DummyDiscoveryService implements DiscoveryService { - private final TbServiceInfoProvider serviceInfoProvider; - private final PartitionService partitionService; + @Autowired + private ServerInstanceService serverInstance; + + @PostConstruct + public void init() { + log.info("Initializing..."); + } + + @Override + public void publishCurrentServer() { + //Do nothing + } + @Override + public void unpublishCurrentServer() { + //Do nothing + } - public DummyDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { - this.serviceInfoProvider = serviceInfoProvider; - this.partitionService = partitionService; + @Override + public ServerInstance getCurrentServer() { + return serverInstance.getSelf(); } - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent event) { - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList()); + @Override + public List getOtherServers() { + return Collections.emptyList(); } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java new file mode 100644 index 0000000000..e9325800af --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.discovery; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.thingsboard.server.common.msg.cluster.ServerAddress; + +/** + * @author Andrew Shvayka + */ +@ToString +@EqualsAndHashCode(exclude = {"serverInfo", "serverAddress"}) +public final class ServerInstance implements Comparable { + + @Getter + private final String host; + @Getter + private final int port; + @Getter + private final ServerAddress serverAddress; + + public ServerInstance(ServerAddress serverAddress) { + this.serverAddress = serverAddress; + this.host = serverAddress.getHost(); + this.port = serverAddress.getPort(); + } + + @Override + public int compareTo(ServerInstance o) { + return this.serverAddress.compareTo(o.serverAddress); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java rename to application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java index d23cc3581d..095e10f0b3 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java @@ -13,10 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.subscription; +package org.thingsboard.server.service.cluster.discovery; -public enum TbAttributeSubscriptionScope { - - CLIENT_SCOPE, SHARED_SCOPE, SERVER_SCOPE +/** + * @author Andrew Shvayka + */ +public interface ServerInstanceService { + ServerInstance getSelf(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java similarity index 67% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java rename to application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java index e761a229c4..e1cdd5f83e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java @@ -13,10 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.service.cluster.discovery; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.SerializationException; +import org.apache.commons.lang3.SerializationUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.imps.CuratorFrameworkState; @@ -30,14 +32,22 @@ import org.apache.curator.retry.RetryForever; import org.apache.curator.utils.CloseableUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; +import org.thingsboard.server.utils.MiscUtils; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -49,6 +59,9 @@ import java.util.stream.Collectors; import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED; +/** + * @author Andrew Shvayka + */ @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) @Slf4j @@ -65,29 +78,42 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi @Value("${zk.zk_dir}") private String zkDir; - private final TbServiceInfoProvider serviceInfoProvider; - private final PartitionService partitionService; + private String zkNodesDir; + + @Autowired + private ServerInstanceService serverInstance; + + @Autowired + @Lazy + private TelemetrySubscriptionService tsSubService; + + @Autowired + @Lazy + private DeviceStateService deviceStateService; + + @Autowired + @Lazy + private ActorService actorService; + + @Autowired + @Lazy + private ClusterRoutingService routingService; private ExecutorService reconnectExecutorService; + private CuratorFramework client; private PathChildrenCache cache; private String nodePath; - private String zkNodesDir; private volatile boolean stopped = true; - public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { - this.serviceInfoProvider = serviceInfoProvider; - this.partitionService = partitionService; - } - @PostConstruct public void init() { log.info("Initializing..."); - Assert.hasLength(zkUrl, missingProperty("zk.url")); - Assert.notNull(zkRetryInterval, missingProperty("zk.retry_interval_ms")); - Assert.notNull(zkConnectionTimeout, missingProperty("zk.connection_timeout_ms")); - Assert.notNull(zkSessionTimeout, missingProperty("zk.session_timeout_ms")); + Assert.hasLength(zkUrl, MiscUtils.missingProperty("zk.url")); + Assert.notNull(zkRetryInterval, MiscUtils.missingProperty("zk.retry_interval_ms")); + Assert.notNull(zkConnectionTimeout, MiscUtils.missingProperty("zk.connection_timeout_ms")); + Assert.notNull(zkSessionTimeout, MiscUtils.missingProperty("zk.session_timeout_ms")); reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); @@ -97,47 +123,53 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi initZkClient(); } - private List getOtherServers() { - return cache.getCurrentData().stream() - .filter(cd -> !cd.getPath().equals(nodePath)) - .map(cd -> { - try { - return TransportProtos.ServiceInfo.parseFrom(cd.getData()); - } catch (NoSuchElementException | InvalidProtocolBufferException e) { - log.error("Failed to decode ZK node", e); - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); + private void initZkClient() { + try { + client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); + client.start(); + client.blockUntilConnected(); + cache = new PathChildrenCache(client, zkNodesDir, true); + cache.getListenable().addListener(this); + cache.start(); + stopped = false; + log.info("ZK client connected"); + } catch (Exception e) { + log.error("Failed to connect to ZK: {}", e.getMessage(), e); + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + throw new RuntimeException(e); + } } - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent event) { - if (stopped) { - log.debug("Ignoring application ready event. Service is stopped."); - return; - } else { - log.info("Received application ready event. Starting current ZK node."); - } - if (client.getState() != CuratorFrameworkState.STARTED) { - log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); - return; - } - publishCurrentServer(); - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); + private void destroyZkClient() { + stopped = true; + try { + unpublishCurrentServer(); + } catch (Exception e) {} + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + log.info("ZK client disconnected"); + } + + @PreDestroy + public void destroy() { + destroyZkClient(); + reconnectExecutorService.shutdownNow(); + log.info("Stopped discovery service"); } + @Override public synchronized void publishCurrentServer() { - TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); + ServerInstance self = this.serverInstance.getSelf(); if (currentServerExists()) { - log.info("[{}] ZK node for current instance already exists, NOT created new one: {}", self.getServiceId(), nodePath); + log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); } else { try { - log.info("[{}] Creating ZK node for current instance", self.getServiceId()); + log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); nodePath = client.create() .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", self.toByteArray()); - log.info("[{}] Created ZK node for current instance: {}", self.getServiceId(), nodePath); + .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); + log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); client.getConnectionStateListenable().addListener(checkReconnect(self)); } catch (Exception e) { log.error("Failed to create ZK node", e); @@ -151,10 +183,10 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return false; } try { - TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); - TransportProtos.ServiceInfo registeredServerInfo = null; - registeredServerInfo = TransportProtos.ServiceInfo.parseFrom(client.getData().forPath(nodePath)); - if (self.equals(registeredServerInfo)) { + ServerInstance self = this.serverInstance.getSelf(); + ServerAddress registeredServerAdress = null; + registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); + if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { return true; } } catch (KeeperException.NoNodeException e) { @@ -165,9 +197,9 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return false; } - private ConnectionStateListener checkReconnect(TransportProtos.ServiceInfo self) { + private ConnectionStateListener checkReconnect(ServerInstance self) { return (client, newState) -> { - log.info("[{}] ZK state changed: {}", self.getServiceId(), newState); + log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); if (newState == ConnectionState.LOST) { reconnectExecutorService.submit(this::reconnect); } @@ -191,25 +223,8 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } - private void initZkClient() { - try { - client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); - client.start(); - client.blockUntilConnected(); - cache = new PathChildrenCache(client, zkNodesDir, true); - cache.getListenable().addListener(this); - cache.start(); - stopped = false; - log.info("ZK client connected"); - } catch (Exception e) { - log.error("Failed to connect to ZK: {}", e.getMessage(), e); - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - throw new RuntimeException(e); - } - } - - private void unpublishCurrentServer() { + @Override + public void unpublishCurrentServer() { try { if (nodePath != null) { client.delete().forPath(nodePath); @@ -220,26 +235,41 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } - private void destroyZkClient() { - stopped = true; - try { - unpublishCurrentServer(); - } catch (Exception e) { - } - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - log.info("ZK client disconnected"); + @Override + public ServerInstance getCurrentServer() { + return serverInstance.getSelf(); } - @PreDestroy - public void destroy() { - destroyZkClient(); - reconnectExecutorService.shutdownNow(); - log.info("Stopped discovery service"); + @Override + public List getOtherServers() { + return cache.getCurrentData().stream() + .filter(cd -> !cd.getPath().equals(nodePath)) + .map(cd -> { + try { + return new ServerInstance((ServerAddress) SerializationUtils.deserialize(cd.getData())); + } catch (NoSuchElementException e) { + log.error("Failed to decode ZK node", e); + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); } - public static String missingProperty(String propertyName) { - return "The " + propertyName + " property need to be set!"; + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + log.info("Received application ready event. Starting current ZK node."); + if (stopped) { + log.debug("Ignoring application ready event. Service is stopped."); + return; + } + if (client.getState() != CuratorFrameworkState.STARTED) { + log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); + return; + } + publishCurrentServer(); + getOtherServers().forEach( + server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) + ); } @Override @@ -267,23 +297,34 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi log.debug("Ignoring event about current server {}", pathChildrenCacheEvent); return; } - TransportProtos.ServiceInfo instance; + ServerInstance instance; try { - instance = TransportProtos.ServiceInfo.parseFrom(data.getData()); - } catch (InvalidProtocolBufferException e) { + ServerAddress serverAddress = SerializationUtils.deserialize(data.getData()); + instance = new ServerInstance(serverAddress); + } catch (SerializationException e) { log.error("Failed to decode server instance for node {}", data.getPath(), e); throw e; } - log.info("Processing [{}] event for [{}]", pathChildrenCacheEvent.getType(), instance.getServiceId()); + log.info("Processing [{}] event for [{}:{}]", pathChildrenCacheEvent.getType(), instance.getHost(), instance.getPort()); switch (pathChildrenCacheEvent.getType()) { case CHILD_ADDED: + routingService.onServerAdded(instance); + tsSubService.onClusterUpdate(); + deviceStateService.onClusterUpdate(); + actorService.onServerAdded(instance); + break; case CHILD_UPDATED: + routingService.onServerUpdated(instance); + actorService.onServerUpdated(instance); + break; case CHILD_REMOVED: - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); + routingService.onServerRemoved(instance); + tsSubService.onClusterUpdate(); + deviceStateService.onClusterUpdate(); + actorService.onServerRemoved(instance); break; default: break; } } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java similarity index 54% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java rename to application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java index fcc7c7a60d..e0ed64fdbd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java @@ -13,22 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.service.cluster.routing; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; import java.util.Optional; +import java.util.UUID; -public interface TbServiceInfoProvider { - - String getServiceId(); - - ServiceInfo getServiceInfo(); +/** + * @author Andrew Shvayka + */ +public interface ClusterRoutingService extends DiscoveryServiceListener { - boolean isService(ServiceType serviceType); + ServerAddress getCurrentServer(); - Optional getIsolatedTenant(); + Optional resolveById(EntityId entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java new file mode 100644 index 0000000000..e114a9b6ab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java @@ -0,0 +1,153 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.routing; + +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.service.cluster.discovery.DiscoveryService; +import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; +import org.thingsboard.server.service.cluster.discovery.ServerInstance; +import org.thingsboard.server.utils.MiscUtils; + +import javax.annotation.PostConstruct; +import java.util.Arrays; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * Cluster service implementation based on consistent hash ring + */ + +@Service +@Slf4j +public class ConsistentClusterRoutingService implements ClusterRoutingService { + + @Autowired + private DiscoveryService discoveryService; + + @Value("${cluster.hash_function_name}") + private String hashFunctionName; + @Value("${cluster.vitrual_nodes_size}") + private Integer virtualNodesSize; + + private ServerInstance currentServer; + + private HashFunction hashFunction; + + private ConsistentHashCircle[] circles; + private ConsistentHashCircle rootCircle; + + @PostConstruct + public void init() { + log.info("Initializing Cluster routing service!"); + this.hashFunction = MiscUtils.forName(hashFunctionName); + this.currentServer = discoveryService.getCurrentServer(); + this.circles = new ConsistentHashCircle[ServerType.values().length]; + for (ServerType serverType : ServerType.values()) { + circles[serverType.ordinal()] = new ConsistentHashCircle(); + } + rootCircle = circles[ServerType.CORE.ordinal()]; + addNode(discoveryService.getCurrentServer()); + for (ServerInstance instance : discoveryService.getOtherServers()) { + addNode(instance); + } + logCircle(); + log.info("Cluster routing service initialized!"); + } + + @Override + public ServerAddress getCurrentServer() { + return discoveryService.getCurrentServer().getServerAddress(); + } + + @Override + public Optional resolveById(EntityId entityId) { + return resolveByUuid(rootCircle, entityId.getId()); + } + + private Optional resolveByUuid(ConsistentHashCircle circle, UUID uuid) { + Assert.notNull(uuid); + if (circle.isEmpty()) { + return Optional.empty(); + } + Long hash = hashFunction.newHasher().putLong(uuid.getMostSignificantBits()) + .putLong(uuid.getLeastSignificantBits()).hash().asLong(); + if (!circle.containsKey(hash)) { + ConcurrentNavigableMap tailMap = + circle.tailMap(hash); + hash = tailMap.isEmpty() ? + circle.firstKey() : tailMap.firstKey(); + } + ServerInstance result = circle.get(hash); + if (!currentServer.equals(result)) { + return Optional.of(result.getServerAddress()); + } else { + return Optional.empty(); + } + } + + @Override + public void onServerAdded(ServerInstance server) { + log.info("On server added event: {}", server); + addNode(server); + logCircle(); + } + + @Override + public void onServerUpdated(ServerInstance server) { + log.debug("Ignoring server onUpdate event: {}", server); + } + + @Override + public void onServerRemoved(ServerInstance server) { + log.info("On server removed event: {}", server); + removeNode(server); + logCircle(); + } + + private void addNode(ServerInstance instance) { + for (int i = 0; i < virtualNodesSize; i++) { + circles[instance.getServerAddress().getServerType().ordinal()].put(hash(instance, i).asLong(), instance); + } + } + + private void removeNode(ServerInstance instance) { + for (int i = 0; i < virtualNodesSize; i++) { + circles[instance.getServerAddress().getServerType().ordinal()].remove(hash(instance, i).asLong()); + } + } + + private HashCode hash(ServerInstance instance, int i) { + return hashFunction.newHasher().putString(instance.getHost(), MiscUtils.UTF8).putInt(instance.getPort()).putInt(i).hash(); + } + + private void logCircle() { + log.trace("Consistent Hash Circle Start"); + Arrays.asList(circles).forEach(ConsistentHashCircle::log); + log.trace("Consistent Hash Circle End"); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java similarity index 69% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java rename to application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java index 1b40545cde..711677dc78 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.service.cluster.routing; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.service.cluster.discovery.ServerInstance; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -24,10 +25,11 @@ import java.util.concurrent.ConcurrentSkipListMap; * Created by ashvayka on 23.09.18. */ @Slf4j -public class ConsistentHashCircle { - private final ConcurrentNavigableMap circle = new ConcurrentSkipListMap<>(); +public class ConsistentHashCircle { + private final ConcurrentNavigableMap circle = + new ConcurrentSkipListMap<>(); - public void put(long hash, T instance) { + public void put(long hash, ServerInstance instance) { circle.put(hash, instance); } @@ -43,7 +45,7 @@ public class ConsistentHashCircle { return circle.containsKey(hash); } - public ConcurrentNavigableMap tailMap(Long hash) { + public ConcurrentNavigableMap tailMap(Long hash) { return circle.tailMap(hash); } @@ -51,11 +53,11 @@ public class ConsistentHashCircle { return circle.firstKey(); } - public T get(Long hash) { + public ServerInstance get(Long hash) { return circle.get(hash); } public void log() { - circle.forEach((key, value) -> log.debug("{} -> {}", key, value)); + circle.entrySet().forEach((e) -> log.debug("{} -> {}", e.getKey(), e.getValue().getServerAddress())); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java new file mode 100644 index 0000000000..23b050634c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.rpc; + +import com.google.protobuf.ByteString; +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; +import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; +import org.thingsboard.server.service.cluster.discovery.ServerInstance; +import org.thingsboard.server.service.cluster.discovery.ServerInstanceService; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; + +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author Andrew Shvayka + */ +@Service +@Slf4j +public class ClusterGrpcService extends ClusterRpcServiceGrpc.ClusterRpcServiceImplBase implements ClusterRpcService { + + @Autowired + private ServerInstanceService instanceService; + + @Autowired + private DataDecodingEncodingService encodingService; + + private RpcMsgListener listener; + + private Server server; + + private ServerInstance instance; + + private ConcurrentMap>> pendingSessionMap = + new ConcurrentHashMap<>(); + + public void init(RpcMsgListener listener) { + this.listener = listener; + log.info("Initializing RPC service!"); + instance = instanceService.getSelf(); + server = ServerBuilder.forPort(instance.getPort()).addService(this).build(); + log.info("Going to start RPC server using port: {}", instance.getPort()); + try { + server.start(); + } catch (IOException e) { + log.error("Failed to start RPC server!", e); + throw new RuntimeException("Failed to start RPC server!"); + } + log.info("RPC service initialized!"); + } + + @Override + public void onSessionCreated(UUID msgUid, StreamObserver inputStream) { + BlockingQueue> queue = pendingSessionMap.remove(msgUid); + if (queue != null) { + try { + queue.put(inputStream); + } catch (InterruptedException e) { + log.warn("Failed to report created session!"); + Thread.currentThread().interrupt(); + } + } else { + log.warn("Failed to lookup pending session!"); + } + } + + @Override + public StreamObserver handleMsgs( + StreamObserver responseObserver) { + log.info("Processing new session."); + return createSession(new RpcSessionCreateRequestMsg(UUID.randomUUID(), null, responseObserver)); + } + + + @PreDestroy + public void stop() { + if (server != null) { + log.info("Going to onStop RPC server"); + server.shutdownNow(); + try { + server.awaitTermination(); + log.info("RPC server stopped!"); + } catch (InterruptedException e) { + log.warn("Failed to onStop RPC server!"); + Thread.currentThread().interrupt(); + } + } + } + + + @Override + public void broadcast(RpcBroadcastMsg msg) { + listener.onBroadcastMsg(msg); + } + + private StreamObserver createSession(RpcSessionCreateRequestMsg msg) { + BlockingQueue> queue = new ArrayBlockingQueue<>(1); + pendingSessionMap.put(msg.getMsgUid(), queue); + listener.onRpcSessionCreateRequestMsg(msg); + try { + StreamObserver observer = queue.take(); + log.info("Processed new session."); + return observer; + } catch (Exception e) { + log.info("Failed to process session.", e); + throw new RuntimeException(e); + } + } + + @Override + public void tell(ClusterAPIProtos.ClusterMessage message) { + listener.onSendMsg(message); + } + + @Override + public void tell(ServerAddress serverAddress, TbActorMsg actorMsg) { + listener.onSendMsg(encodingService.convertToProtoDataMessage(serverAddress, actorMsg)); + } + + @Override + public void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data) { + ClusterAPIProtos.ClusterMessage msg = ClusterAPIProtos.ClusterMessage + .newBuilder() + .setServerAddress(ClusterAPIProtos.ServerAddress + .newBuilder() + .setHost(serverAddress.getHost()) + .setPort(serverAddress.getPort()) + .build()) + .setMessageType(msgType) + .setPayload(ByteString.copyFrom(data)).build(); + listener.onSendMsg(msg); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java new file mode 100644 index 0000000000..637924fca9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.rpc; + +import io.grpc.stub.StreamObserver; +import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; + +import java.util.UUID; + +/** + * @author Andrew Shvayka + */ +public interface ClusterRpcService { + + void init(RpcMsgListener listener); + + void broadcast(RpcBroadcastMsg msg); + + void onSessionCreated(UUID msgUid, StreamObserver inputStream); + + void tell(ClusterAPIProtos.ClusterMessage message); + + void tell(ServerAddress serverAddress, TbActorMsg actorMsg); + + void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data); +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java new file mode 100644 index 0000000000..aff0bc4a41 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.rpc; + +import io.grpc.Channel; +import io.grpc.ManagedChannel; +import io.grpc.stub.StreamObserver; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; + +import java.io.Closeable; +import java.util.UUID; + +/** + * @author Andrew Shvayka + */ +@Data +@Slf4j +public final class GrpcSession implements Closeable { + private final UUID sessionId; + private final boolean client; + private final GrpcSessionListener listener; + private final ManagedChannel channel; + private StreamObserver inputStream; + private StreamObserver outputStream; + + private boolean connected; + private ServerAddress remoteServer; + + public GrpcSession(GrpcSessionListener listener) { + this(null, listener, null); + } + + public GrpcSession(ServerAddress remoteServer, GrpcSessionListener listener, ManagedChannel channel) { + this.sessionId = UUID.randomUUID(); + this.listener = listener; + if (remoteServer != null) { + this.client = true; + this.connected = true; + this.remoteServer = remoteServer; + } else { + this.client = false; + } + this.channel = channel; + } + + public void initInputStream() { + this.inputStream = new StreamObserver() { + @Override + public void onNext(ClusterAPIProtos.ClusterMessage clusterMessage) { + if (!connected && clusterMessage.getMessageType() == ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE) { + connected = true; + ServerAddress rpcAddress = new ServerAddress(clusterMessage.getServerAddress().getHost(), clusterMessage.getServerAddress().getPort(), ServerType.CORE); + remoteServer = new ServerAddress(rpcAddress.getHost(), rpcAddress.getPort(), ServerType.CORE); + listener.onConnected(GrpcSession.this); + } + if (connected) { + listener.onReceiveClusterGrpcMsg(GrpcSession.this, clusterMessage); + } + } + + @Override + public void onError(Throwable t) { + listener.onError(GrpcSession.this, t); + } + + @Override + public void onCompleted() { + outputStream.onCompleted(); + listener.onDisconnected(GrpcSession.this); + } + }; + } + + public void initOutputStream() { + if (client) { + listener.onConnected(GrpcSession.this); + } + } + + public void sendMsg(ClusterAPIProtos.ClusterMessage msg) { + if (connected) { + try { + outputStream.onNext(msg); + } catch (Throwable t) { + try { + outputStream.onError(t); + } catch (Throwable t2) { + } + listener.onError(GrpcSession.this, t); + } + } else { + log.warn("[{}] Failed to send message due to closed session!", sessionId); + } + } + + @Override + public void close() { + connected = false; + try { + outputStream.onCompleted(); + } catch (IllegalStateException e) { + log.debug("[{}] Failed to close output stream: {}", sessionId, e.getMessage()); + } + if (channel != null) { + channel.shutdownNow(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java similarity index 59% rename from application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java rename to application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java index 3d4f23a796..a6ecf967d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.rpc; +package org.thingsboard.server.service.cluster.rpc; -import org.thingsboard.rule.engine.api.RuleEngineRpcService; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; /** - * Created by ashvayka on 16.04.18. + * @author Andrew Shvayka */ -public interface TbRuleEngineDeviceRpcService extends RuleEngineRpcService { +public interface GrpcSessionListener { - /** - * Handles the RPC response from the Device Actor (Transport). - * - * @param response the RPC response - */ - void processRpcResponseFromDevice(FromDeviceRpcResponse response); + void onConnected(GrpcSession session); + void onDisconnected(GrpcSession session); + + void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage); + + void onError(GrpcSession session, Throwable t); } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java new file mode 100644 index 0000000000..2ff37b39e0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.rpc; + +import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; +import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; + +/** + * @author Andrew Shvayka + */ + +public interface RpcMsgListener { + void onReceivedMsg(ServerAddress remoteServer, ClusterAPIProtos.ClusterMessage msg); + void onSendMsg(ClusterAPIProtos.ClusterMessage msg); + void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg); + void onBroadcastMsg(RpcBroadcastMsg msg); +} diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java index 4a781b8673..a5d3ab465e 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.encoding; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; @@ -25,5 +27,8 @@ public interface DataDecodingEncodingService { byte[] encode(TbActorMsg msq); + ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, + TbActorMsg msg); + } diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java index 8d89059488..45bb9f78fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java @@ -15,19 +15,25 @@ */ package org.thingsboard.server.service.encoding; +import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; +import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; + + @Slf4j @Service public class ProtoWithFSTService implements DataDecodingEncodingService { - private final FSTConfiguration config = FSTConfiguration.createDefaultConfiguration(); + private final FSTConfiguration config = FSTConfiguration.createDefaultConfiguration(); @Override public Optional decode(byte[] byteArray) { try { @@ -36,7 +42,7 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { } catch (IllegalArgumentException e) { log.error("Error during deserialization message, [{}]", e.getMessage()); - return Optional.empty(); + return Optional.empty(); } } @@ -45,4 +51,18 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { return config.asByteArray(msq); } + @Override + public ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, + TbActorMsg msg) { + return ClusterAPIProtos.ClusterMessage + .newBuilder() + .setServerAddress(ClusterAPIProtos.ServerAddress + .newBuilder() + .setHost(serverAddress.getHost()) + .setPort(serverAddress.getPort()) + .build()) + .setMessageType(CLUSTER_ACTOR_MESSAGE) + .setPayload(ByteString.copyFrom(encode(msg))).build(); + + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java b/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java similarity index 90% rename from common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java rename to application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java index 267b34b204..a6a881ecde 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java +++ b/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.environment; +package org.thingsboard.server.service.environment; import lombok.extern.slf4j.Slf4j; import org.apache.zookeeper.Environment; @@ -33,7 +33,7 @@ public class EnvironmentLogService { @PostConstruct public void init() { - Environment.logEnv("ThingsBoard server environment: ", log); + Environment.logEnv("Thingsboard server environment: ", log); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java b/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java similarity index 63% rename from common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java rename to application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java index 50da4f4a1e..f4b14144fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java @@ -13,20 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.settings; +package org.thingsboard.server.service.executors; -import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.AbstractListeningExecutor; -@Data @Component -public class TbQueueTransportNotificationSettings { +public class ClusterRpcCallbackExecutorService extends AbstractListeningExecutor { - @Value("${queue.transport.notifications_topic}") - private String notificationsTopic; + @Value("${actors.cluster.grpc_callback_thread_pool_size}") + private int grpcCallbackExecutorThreadPoolSize; - @Value("${queue.transport.poll_interval}") - private long transportPollInterval; + @Override + protected int getThreadPollSize() { + return grpcCallbackExecutorThreadPoolSize; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 87164737c7..ef03e3ec43 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,15 +221,6 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } - try { - conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); - } catch (Exception e) { - } - try { - long ts = System.currentTimeMillis(); - conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script - } catch (Exception e) { - } log.info("Schema updated."); } break; diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java index e7f21a18d0..08a5fdd3e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java @@ -203,8 +203,7 @@ public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateServ stringColumn("entity_type"), stringColumn("event_type"), stringColumn("event_uid"), - stringColumn("body"), - new CassandraToSqlEventTsColumn()), + stringColumn("body")), new CassandraToSqlTable("relation", idColumn("from_id"), stringColumn("from_type"), @@ -246,9 +245,7 @@ public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateServ stringColumn("zip"), stringColumn("phone"), stringColumn("email"), - stringColumn("additional_info"), - booleanColumn("isolated_tb_core"), - booleanColumn("isolated_tb_rule_engine")), + stringColumn("additional_info")), new CassandraToSqlTable("user_credentials", idColumn("id"), idColumn("user_id"), diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java deleted file mode 100644 index 8e369c66c6..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlEventTsColumn.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.install.migrate; - -import com.datastax.driver.core.Row; - -import java.util.UUID; - -import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; - -public class CassandraToSqlEventTsColumn extends CassandraToSqlColumn { - - CassandraToSqlEventTsColumn() { - super("id", "ts", CassandraToSqlColumnType.BIGINT, null); - } - - @Override - public String getColumnValue(Row row) { - UUID id = row.getUUID(getIndex()); - long ts = getTs(id); - return ts + ""; - } - - private long getTs(UUID uuid) { - return (uuid.timestamp() - EPOCH_DIFF) / 10000; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java deleted file mode 100644 index 926b08f38b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ /dev/null @@ -1,204 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import com.google.protobuf.ByteString; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; - -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; - -@Service -@Slf4j -public class DefaultTbClusterService implements TbClusterService { - - @Value("${cluster.stats.enabled:false}") - private boolean statsEnabled; - - private final AtomicInteger toCoreMsgs = new AtomicInteger(0); - private final AtomicInteger toCoreNfs = new AtomicInteger(0); - private final AtomicInteger toRuleEngineMsgs = new AtomicInteger(0); - private final AtomicInteger toRuleEngineNfs = new AtomicInteger(0); - private final AtomicInteger toTransportNfs = new AtomicInteger(0); - - private final TbQueueProducerProvider producerProvider; - private final PartitionService partitionService; - private final DataDecodingEncodingService encodingService; - - public DefaultTbClusterService(TbQueueProducerProvider producerProvider, PartitionService partitionService, DataDecodingEncodingService encodingService) { - this.producerProvider = producerProvider; - this.partitionService = partitionService; - this.encodingService = encodingService; - } - - @Override - public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); - producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); - toCoreMsgs.incrementAndGet(); - } - - @Override - public void pushMsgToCore(TopicPartitionInfo tpi, UUID msgId, ToCoreMsg msg, TbQueueCallback callback) { - producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); - toCoreMsgs.incrementAndGet(); - } - - @Override - public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); - log.trace("PUSHING msg: {} to:{}", msg, tpi); - byte[] msgBytes = encodingService.encode(msg); - ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); - producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback); - toCoreMsgs.incrementAndGet(); - } - - @Override - public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); - log.trace("PUSHING msg: {} to:{}", response, tpi); - FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() - .setRequestIdMSB(response.getId().getMostSignificantBits()) - .setRequestIdLSB(response.getId().getLeastSignificantBits()) - .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); - response.getResponse().ifPresent(builder::setResponse); - ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); - producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); - toCoreNfs.incrementAndGet(); - } - - @Override - public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) { - log.trace("PUSHING msg: {} to:{}", msg, tpi); - producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); - toRuleEngineMsgs.incrementAndGet(); - } - - @Override - public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg tbMsg, TbQueueCallback callback) { - if (tenantId.isNullUid()) { - if (entityId.getEntityType().equals(EntityType.TENANT)) { - tenantId = new TenantId(entityId.getId()); - } else { - log.warn("[{}][{}] Received invalid message: {}", tenantId, entityId, tbMsg); - return; - } - } - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - log.trace("PUSHING msg: {} to:{}", tbMsg, tpi); - ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); - toRuleEngineMsgs.incrementAndGet(); - } - - @Override - public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); - log.trace("PUSHING msg: {} to:{}", response, tpi); - FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() - .setRequestIdMSB(response.getId().getMostSignificantBits()) - .setRequestIdLSB(response.getId().getLeastSignificantBits()) - .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); - response.getResponse().ifPresent(builder::setResponse); - ToRuleEngineNotificationMsg msg = ToRuleEngineNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); - producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); - toRuleEngineNfs.incrementAndGet(); - } - - @Override - public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); - log.trace("PUSHING msg: {} to:{}", response, tpi); - producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback); - toTransportNfs.incrementAndGet(); - } - - @Override - public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { - log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); - broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); - } - - private void broadcast(ComponentLifecycleMsg msg) { - byte[] msgBytes = encodingService.encode(msg); - TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); - Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); - if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)) { - TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); - Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); - for (String serviceId : tbCoreServices) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); - ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); - toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); - toCoreNfs.incrementAndGet(); - } - // No need to push notifications twice - tbRuleEngineServices.removeAll(tbCoreServices); - } - for (String serviceId : tbRuleEngineServices) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); - ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); - toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null); - toRuleEngineNfs.incrementAndGet(); - } - } - - @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - int toCoreMsgCnt = toCoreMsgs.getAndSet(0); - int toCoreNfsCnt = toCoreNfs.getAndSet(0); - int toRuleEngineMsgsCnt = toRuleEngineMsgs.getAndSet(0); - int toRuleEngineNfsCnt = toRuleEngineNfs.getAndSet(0); - int toTransportNfsCnt = toTransportNfs.getAndSet(0); - if (toCoreMsgCnt > 0 || toCoreNfsCnt > 0 || toRuleEngineMsgsCnt > 0 || toRuleEngineNfsCnt > 0 || toTransportNfsCnt > 0) { - log.info("To TbCore: [{}] messages [{}] notifications; To TbRuleEngine: [{}] messages [{}] notifications; To Transport: [{}] notifications", - toCoreMsgCnt, toCoreNfsCnt, toRuleEngineMsgsCnt, toRuleEngineNfsCnt, toTransportNfsCnt); - } - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java deleted file mode 100644 index a5abd77ad5..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ /dev/null @@ -1,290 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; -import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.provider.TbCoreQueueFactory; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.queue.processing.AbstractConsumerService; -import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.subscription.SubscriptionManagerService; -import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; -import org.thingsboard.server.service.subscription.TbSubscriptionUtils; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Service -@TbCoreComponent -@Slf4j -public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService { - - @Value("${queue.core.poll-interval}") - private long pollDuration; - @Value("${queue.core.pack-processing-timeout}") - private long packProcessingTimeout; - @Value("${queue.core.stats.enabled:false}") - private boolean statsEnabled; - - private final TbQueueConsumer> mainConsumer; - private final DeviceStateService stateService; - private final TbLocalSubscriptionService localSubscriptionService; - private final SubscriptionManagerService subscriptionManagerService; - private final TbCoreDeviceRpcService tbCoreDeviceRpcService; - private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); - - public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, - DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, - SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, - TbCoreDeviceRpcService tbCoreDeviceRpcService) { - super(actorContext, encodingService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); - this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); - this.stateService = stateService; - this.localSubscriptionService = localSubscriptionService; - this.subscriptionManagerService = subscriptionManagerService; - this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; - } - - @PostConstruct - public void init() { - super.init("tb-core-consumer", "tb-core-notifications-consumer"); - } - - @PreDestroy - public void destroy(){ - super.destroy(); - } - - @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (partitionChangeEvent.getServiceType().equals(getServiceType())) { - log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); - } - } - - @Override - protected void launchMainConsumers() { - consumersExecutor.submit(() -> { - while (!stopped) { - try { - List> msgs = mainConsumer.poll(pollDuration); - if (msgs.isEmpty()) { - continue; - } - ConcurrentMap> pendingMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - TbPackProcessingContext> ctx = new TbPackProcessingContext<>( - processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - pendingMap.forEach((id, msg) -> { - log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); - TbCallback callback = new TbPackCallback<>(id, ctx); - try { - ToCoreMsg toCoreMsg = msg.getValue(); - if (toCoreMsg.hasToSubscriptionMgrMsg()) { - log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); - forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); - } else if (toCoreMsg.hasToDeviceActorMsg()) { - log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); - forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); - } else if (toCoreMsg.hasDeviceStateServiceMsg()) { - log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); - forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); - } else if (toCoreMsg.getToDeviceActorNotificationMsg() != null && !toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); - if (actorMsg.isPresent()) { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.tell(actorMsg.get(), ActorRef.noSender()); - } - callback.onSuccess(); - } - } catch (Throwable e) { - log.warn("[{}] Failed to process message: {}", id, msg, e); - callback.onFailure(e); - } - }); - if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { - ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); - ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); - } - mainConsumer.commit(); - } catch (Exception e) { - if (!stopped) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - } - log.info("TB Core Consumer stopped."); - }); - } - - @Override - protected ServiceType getServiceType() { - return ServiceType.TB_CORE; - } - - @Override - protected long getNotificationPollDuration() { - return pollDuration; - } - - @Override - protected long getNotificationPackProcessingTimeout() { - return packProcessingTimeout; - } - - @Override - protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { - ToCoreNotificationMsg toCoreNotification = msg.getValue(); - if (toCoreNotification.hasToLocalSubscriptionServiceMsg()) { - log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreNotification.getToLocalSubscriptionServiceMsg()); - forwardToLocalSubMgrService(toCoreNotification.getToLocalSubscriptionServiceMsg(), callback); - } else if (toCoreNotification.hasFromDeviceRpcResponse()) { - log.trace("[{}] Forwarding message to RPC service {}", id, toCoreNotification.getFromDeviceRpcResponse()); - forwardToCoreRpcService(toCoreNotification.getFromDeviceRpcResponse(), callback); - } else if (toCoreNotification.getComponentLifecycleMsg() != null && !toCoreNotification.getComponentLifecycleMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(toCoreNotification.getComponentLifecycleMsg().toByteArray()); - if (actorMsg.isPresent()) { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.tell(actorMsg.get(), ActorRef.noSender()); - } - callback.onSuccess(); - } - if (statsEnabled) { - stats.log(toCoreNotification); - } - } - - private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { - RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; - FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) - , proto.getResponse(), error); - tbCoreDeviceRpcService.processRpcResponseFromRuleEngine(response); - callback.onSuccess(); - } - - @Scheduled(fixedDelayString = "${queue.core.stats.print-interval-ms}") - public void printStats() { - if (statsEnabled) { - stats.printStats(); - } - } - - private void forwardToLocalSubMgrService(LocalSubscriptionServiceMsgProto msg, TbCallback callback) { - if (msg.hasSubUpdate()) { - localSubscriptionService.onSubscriptionUpdate(msg.getSubUpdate().getSessionId(), TbSubscriptionUtils.fromProto(msg.getSubUpdate()), callback); - } else { - throwNotHandled(msg, callback); - } - } - - private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbCallback callback) { - if (msg.hasAttributeSub()) { - subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getAttributeSub()), callback); - } else if (msg.hasTelemetrySub()) { - subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getTelemetrySub()), callback); - } else if (msg.hasSubClose()) { - TbSubscriptionCloseProto closeProto = msg.getSubClose(); - subscriptionManagerService.cancelSubscription(closeProto.getSessionId(), closeProto.getSubscriptionId(), callback); - } else if (msg.hasTsUpdate()) { - TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); - subscriptionManagerService.onTimeSeriesUpdate( - new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), - TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), - TbSubscriptionUtils.toTsKvEntityList(proto.getDataList()), callback); - } else if (msg.hasAttrUpdate()) { - TbAttributeUpdateProto proto = msg.getAttrUpdate(); - subscriptionManagerService.onAttributesUpdate( - new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), - TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), - proto.getScope(), TbSubscriptionUtils.toAttributeKvList(proto.getDataList()), callback); - } else { - throwNotHandled(msg, callback); - } - if (statsEnabled) { - stats.log(msg); - } - } - - private void forwardToStateService(DeviceStateServiceMsgProto deviceStateServiceMsg, TbCallback callback) { - if (statsEnabled) { - stats.log(deviceStateServiceMsg); - } - stateService.onQueueMsg(deviceStateServiceMsg, callback); - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbCallback callback) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - actorContext.tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); - } - - private void throwNotHandled(Object msg, TbCallback callback) { - log.warn("Message not handled: {}", msg); - callback.onFailure(new RuntimeException("Message not handled!")); - } - - @Override - protected void stopMainConsumers() { - if (mainConsumer != null) { - mainConsumer.unsubscribe(); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java deleted file mode 100644 index 0c48c54412..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ /dev/null @@ -1,262 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import akka.actor.ActorRef; -import com.google.protobuf.ProtocolStringList; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.ServiceQueue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; -import org.thingsboard.server.queue.util.TbRuleEngineComponent; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.queue.processing.AbstractConsumerService; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; -import org.thingsboard.server.service.stats.RuleEngineStatisticsService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -@Service -@TbRuleEngineComponent -@Slf4j -public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { - - @Value("${queue.rule-engine.poll-interval}") - private long pollDuration; - @Value("${queue.rule-engine.pack-processing-timeout}") - private long packProcessingTimeout; - @Value("${queue.rule-engine.stats.enabled:true}") - private boolean statsEnabled; - - private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; - private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory; - private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final RuleEngineStatisticsService statisticsService; - private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); - private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); - private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); - private ExecutorService submitExecutor; - - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, - TbRuleEngineSubmitStrategyFactory submitStrategyFactory, - TbQueueRuleEngineSettings ruleEngineSettings, - TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, - ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { - super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); - this.statisticsService = statisticsService; - this.ruleEngineSettings = ruleEngineSettings; - this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; - this.submitStrategyFactory = submitStrategyFactory; - this.processingStrategyFactory = processingStrategyFactory; - } - - @PostConstruct - public void init() { - super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); - for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) { - consumerConfigurations.putIfAbsent(configuration.getName(), configuration); - consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); - consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName())); - } - submitExecutor = Executors.newSingleThreadExecutor(); - } - - @PreDestroy - public void stop() { - super.destroy(); - if (submitExecutor != null) { - submitExecutor.shutdownNow(); - } - ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config)); - } - - @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (partitionChangeEvent.getServiceType().equals(getServiceType())) { - ServiceQueue serviceQueue = partitionChangeEvent.getServiceQueueKey().getServiceQueue(); - log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), partitionChangeEvent.getPartitions()); - consumers.get(serviceQueue.getQueue()).subscribe(partitionChangeEvent.getPartitions()); - } - } - - @Override - protected void launchMainConsumers() { - consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue))); - } - - @Override - protected void stopMainConsumers() { - consumers.values().forEach(TbQueueConsumer::unsubscribe); - } - - private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats) { - consumersExecutor.execute(() -> { - while (!stopped) { - try { - List> msgs = consumer.poll(pollDuration); - if (msgs.isEmpty()) { - continue; - } - TbRuleEngineSubmitStrategy submitStrategy = submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy()); - TbRuleEngineProcessingStrategy ackStrategy = processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy()); - - submitStrategy.init(msgs); - - while (!stopped) { - TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(submitStrategy); - submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> { - log.trace("[{}] Creating callback for message: {}", id, msg.getValue()); - ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); - TbMsgCallback callback = new TbMsgPackCallback(id, tenantId, ctx); - try { - if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { - forwardToRuleEngineActor(tenantId, toRuleEngineMsg, callback); - } else { - callback.onSuccess(); - } - } catch (Exception e) { - callback.onFailure(new RuleEngineException(e.getMessage())); - } - })); - - boolean timeout = false; - if (!ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { - timeout = true; - } - - TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(timeout, ctx); - TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); - if (statsEnabled) { - stats.log(result, decision.isCommit()); - } - if (decision.isCommit()) { - submitStrategy.stop(); - break; - } else { - submitStrategy.update(decision.getReprocessMap()); - } - } - consumer.commit(); - } catch (Exception e) { - if (!stopped) { - log.warn("Failed to process messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - } - log.info("TB Rule Engine Consumer stopped."); - }); - } - - @Override - protected ServiceType getServiceType() { - return ServiceType.TB_RULE_ENGINE; - } - - @Override - protected long getNotificationPollDuration() { - return pollDuration; - } - - @Override - protected long getNotificationPackProcessingTimeout() { - return packProcessingTimeout; - } - - @Override - protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception { - ToRuleEngineNotificationMsg nfMsg = msg.getValue(); - if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(nfMsg.getComponentLifecycleMsg().toByteArray()); - if (actorMsg.isPresent()) { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.tell(actorMsg.get(), ActorRef.noSender()); - } - callback.onSuccess(); - } else { - callback.onSuccess(); - } - } - - private void forwardToRuleEngineActor(TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { - TbMsg tbMsg = TbMsg.fromBytes(toRuleEngineMsg.getTbMsg().toByteArray(), callback); - QueueToRuleEngineMsg msg; - ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); - Set relationTypes = null; - if (relationTypesList != null) { - if (relationTypesList.size() == 1) { - relationTypes = Collections.singleton(relationTypesList.get(0)); - } else { - relationTypes = new HashSet<>(relationTypesList); - } - } - msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); - actorContext.tell(msg, ActorRef.noSender()); - } - - @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") - public void printStats() { - if (statsEnabled) { - long ts = System.currentTimeMillis(); - consumerStats.forEach((queue, stats) -> { - stats.printStats(); - statisticsService.reportQueueStats(ts, stats); - }); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java deleted file mode 100644 index aae3cef0ac..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.discovery.TenantRoutingInfo; -import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; - -@Slf4j -@Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'") -public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService { - - private final TenantService tenantService; - - public DefaultTenantRoutingInfoService(TenantService tenantService) { - this.tenantService = tenantService; - } - - @Override - public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { - Tenant tenant = tenantService.findTenantById(tenantId); - if (tenant != null) { - return new TenantRoutingInfo(tenantId, tenant.isIsolatedTbCore(), tenant.isIsolatedTbRuleEngine()); - } else { - throw new RuntimeException("Tenant not found!"); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java deleted file mode 100644 index cc722720d5..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; - -import java.util.UUID; - -public interface TbClusterService { - - void pushMsgToCore(TopicPartitionInfo tpi, UUID msgKey, ToCoreMsg msg, TbQueueCallback callback); - - void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback); - - void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback); - - void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); - - void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, TransportProtos.ToRuleEngineMsg msg, TbQueueCallback callback); - - void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg msg, TbQueueCallback callback); - - void pushNotificationToRuleEngine(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); - - void pushNotificationToTransport(String targetServiceId, ToTransportMsg response, TbQueueCallback callback); - - void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java deleted file mode 100644 index 2dcdb0e0c2..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import org.springframework.context.ApplicationListener; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; - -public interface TbCoreConsumerService extends ApplicationListener { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java deleted file mode 100644 index f093cc885a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; - -import java.util.UUID; - -@Slf4j -public class TbMsgPackCallback implements TbMsgCallback { - private final UUID id; - private final TenantId tenantId; - private final TbMsgPackProcessingContext ctx; - - public TbMsgPackCallback(UUID id, TenantId tenantId, TbMsgPackProcessingContext ctx) { - this.id = id; - this.tenantId = tenantId; - this.ctx = ctx; - } - - @Override - public void onSuccess() { - log.trace("[{}] ON SUCCESS", id); - ctx.onSuccess(id); - } - - @Override - public void onFailure(RuleEngineException e) { - log.trace("[{}] ON FAILURE", id, e); - ctx.onFailure(tenantId, id, e); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java deleted file mode 100644 index 7e78ac6f5f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.Getter; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; - -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -public class TbMsgPackProcessingContext { - - private final TbRuleEngineSubmitStrategy submitStrategy; - - private final AtomicInteger pendingCount; - private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - @Getter - private final ConcurrentMap> pendingMap; - @Getter - private final ConcurrentMap> successMap = new ConcurrentHashMap<>(); - @Getter - private final ConcurrentMap> failedMap = new ConcurrentHashMap<>(); - @Getter - private final ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); - - public TbMsgPackProcessingContext(TbRuleEngineSubmitStrategy submitStrategy) { - this.submitStrategy = submitStrategy; - this.pendingMap = submitStrategy.getPendingMap(); - this.pendingCount = new AtomicInteger(pendingMap.size()); - } - - public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { - return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); - } - - public void onSuccess(UUID id) { - TbProtoQueueMsg msg; - boolean empty = false; - msg = pendingMap.remove(id); - if (msg != null) { - empty = pendingCount.decrementAndGet() == 0; - successMap.put(id, msg); - submitStrategy.onSuccess(id); - } - if (empty) { - processingTimeoutLatch.countDown(); - } - } - - public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) { - TbProtoQueueMsg msg; - boolean empty = false; - msg = pendingMap.remove(id); - if (msg != null) { - empty = pendingCount.decrementAndGet() == 0; - failedMap.put(id, msg); - exceptionsMap.putIfAbsent(tenantId, e); - } - if (empty) { - processingTimeoutLatch.countDown(); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java deleted file mode 100644 index e9f1224625..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.extern.slf4j.Slf4j; - -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -@Slf4j -public class TbPackProcessingContext { - - private final AtomicInteger pendingCount; - private final CountDownLatch processingTimeoutLatch; - private final ConcurrentMap ackMap; - private final ConcurrentMap failedMap; - - public TbPackProcessingContext(CountDownLatch processingTimeoutLatch, - ConcurrentMap ackMap, - ConcurrentMap failedMap) { - this.processingTimeoutLatch = processingTimeoutLatch; - this.pendingCount = new AtomicInteger(ackMap.size()); - this.ackMap = ackMap; - this.failedMap = failedMap; - } - - public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { - return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); - } - - public void onSuccess(UUID id) { - boolean empty = false; - T msg = ackMap.remove(id); - if (msg != null) { - empty = pendingCount.decrementAndGet() == 0; - } - if (empty) { - processingTimeoutLatch.countDown(); - } else { - if (log.isTraceEnabled()) { - log.trace("Items left: {}", ackMap.size()); - for (T t : ackMap.values()) { - log.trace("left item: {}", t); - } - } - } - } - - public void onFailure(UUID id, Throwable t) { - boolean empty = false; - T msg = ackMap.remove(id); - if (msg != null) { - empty = pendingCount.decrementAndGet() == 0; - failedMap.put(id, msg); - if (log.isTraceEnabled()) { - log.trace("Items left: {}", ackMap.size()); - for (T v : ackMap.values()) { - log.trace("left item: {}", v); - } - } - } - if (empty) { - processingTimeoutLatch.countDown(); - } - } - - public ConcurrentMap getAckMap() { - return ackMap; - } - - public ConcurrentMap getFailedMap() { - return failedMap; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java deleted file mode 100644 index 19966b4566..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import org.springframework.context.ApplicationListener; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; - -public interface TbRuleEngineConsumerService extends ApplicationListener { - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java deleted file mode 100644 index 40017d2b40..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; - -@Slf4j -@Data -public class TbRuleEngineConsumerStats { - - public static final String TOTAL_MSGS = "totalMsgs"; - public static final String SUCCESSFUL_MSGS = "successfulMsgs"; - public static final String TMP_TIMEOUT = "tmpTimeout"; - public static final String TMP_FAILED = "tmpFailed"; - public static final String TIMEOUT_MSGS = "timeoutMsgs"; - public static final String FAILED_MSGS = "failedMsgs"; - public static final String SUCCESSFUL_ITERATIONS = "successfulIterations"; - public static final String FAILED_ITERATIONS = "failedIterations"; - - private final AtomicInteger totalMsgCounter = new AtomicInteger(0); - private final AtomicInteger successMsgCounter = new AtomicInteger(0); - private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); - private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); - - private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); - private final AtomicInteger failedMsgCounter = new AtomicInteger(0); - - private final AtomicInteger successIterationsCounter = new AtomicInteger(0); - private final AtomicInteger failedIterationsCounter = new AtomicInteger(0); - - private final Map counters = new HashMap<>(); - private final ConcurrentMap tenantStats = new ConcurrentHashMap<>(); - private final ConcurrentMap tenantExceptions = new ConcurrentHashMap<>(); - - private final String queueName; - - public TbRuleEngineConsumerStats(String queueName) { - this.queueName = queueName; - counters.put(TOTAL_MSGS, totalMsgCounter); - counters.put(SUCCESSFUL_MSGS, successMsgCounter); - counters.put(TIMEOUT_MSGS, timeoutMsgCounter); - counters.put(FAILED_MSGS, failedMsgCounter); - - counters.put(TMP_TIMEOUT, tmpTimeoutMsgCounter); - counters.put(TMP_FAILED, tmpFailedMsgCounter); - counters.put(SUCCESSFUL_ITERATIONS, successIterationsCounter); - counters.put(FAILED_ITERATIONS, failedIterationsCounter); - } - - public void log(TbRuleEngineProcessingResult msg, boolean finalIterationForPack) { - int success = msg.getSuccessMap().size(); - int pending = msg.getPendingMap().size(); - int failed = msg.getFailedMap().size(); - totalMsgCounter.addAndGet(success + pending + failed); - successMsgCounter.addAndGet(success); - msg.getSuccessMap().values().forEach(m -> getTenantStats(m).logSuccess()); - if (finalIterationForPack) { - if (pending > 0 || failed > 0) { - timeoutMsgCounter.addAndGet(pending); - failedMsgCounter.addAndGet(failed); - if (pending > 0) { - msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTimeout()); - } - if (failed > 0) { - msg.getFailedMap().values().forEach(m -> getTenantStats(m).logFailed()); - } - failedIterationsCounter.incrementAndGet(); - } else { - successIterationsCounter.incrementAndGet(); - } - } else { - failedIterationsCounter.incrementAndGet(); - tmpTimeoutMsgCounter.addAndGet(pending); - tmpFailedMsgCounter.addAndGet(failed); - if (pending > 0) { - msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTmpTimeout()); - } - if (failed > 0) { - msg.getFailedMap().values().forEach(m -> getTenantStats(m).logTmpFailed()); - } - } - msg.getExceptionsMap().forEach(tenantExceptions::putIfAbsent); - } - - private TbTenantRuleEngineStats getTenantStats(TbProtoQueueMsg m) { - ToRuleEngineMsg reMsg = m.getValue(); - return tenantStats.computeIfAbsent(new UUID(reMsg.getTenantIdMSB(), reMsg.getTenantIdLSB()), TbTenantRuleEngineStats::new); - } - - public void printStats() { - int total = totalMsgCounter.get(); - if (total > 0) { - StringBuilder stats = new StringBuilder(); - counters.forEach((label, value) -> { - stats.append(label).append(" = [").append(value.get()).append("] "); - }); - log.info("[{}] Stats: {}", queueName, stats); - } - } - - public void reset() { - counters.values().forEach(counter -> counter.set(0)); - tenantStats.clear(); - tenantExceptions.clear(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java deleted file mode 100644 index fb36f5064b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; - -@Slf4j -@Data -public class TbTenantRuleEngineStats { - - private final UUID tenantId; - - private final AtomicInteger totalMsgCounter = new AtomicInteger(0); - private final AtomicInteger successMsgCounter = new AtomicInteger(0); - private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); - private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); - - private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); - private final AtomicInteger failedMsgCounter = new AtomicInteger(0); - - private final Map counters = new HashMap<>(); - - public TbTenantRuleEngineStats(UUID tenantId) { - this.tenantId = tenantId; - counters.put(TbRuleEngineConsumerStats.TOTAL_MSGS, totalMsgCounter); - counters.put(TbRuleEngineConsumerStats.SUCCESSFUL_MSGS, successMsgCounter); - counters.put(TbRuleEngineConsumerStats.TIMEOUT_MSGS, timeoutMsgCounter); - counters.put(TbRuleEngineConsumerStats.FAILED_MSGS, failedMsgCounter); - - counters.put(TbRuleEngineConsumerStats.TMP_TIMEOUT, tmpTimeoutMsgCounter); - counters.put(TbRuleEngineConsumerStats.TMP_FAILED, tmpFailedMsgCounter); - } - - public void logSuccess() { - totalMsgCounter.incrementAndGet(); - successMsgCounter.incrementAndGet(); - } - - public void logFailed() { - totalMsgCounter.incrementAndGet(); - failedMsgCounter.incrementAndGet(); - } - - public void logTimeout() { - totalMsgCounter.incrementAndGet(); - timeoutMsgCounter.incrementAndGet(); - } - - public void logTmpFailed() { - totalMsgCounter.incrementAndGet(); - tmpFailedMsgCounter.incrementAndGet(); - } - - public void logTmpTimeout() { - totalMsgCounter.incrementAndGet(); - tmpTimeoutMsgCounter.incrementAndGet(); - } - - public void printStats() { - int total = totalMsgCounter.get(); - if (total > 0) { - StringBuilder stats = new StringBuilder(); - counters.forEach((label, value) -> { - stats.append(label).append(" = [").append(value.get()).append("]"); - }); - log.info("[{}] Stats: {}", tenantId, stats); - } - } - - public void reset() { - counters.values().forEach(counter -> counter.set(0)); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java deleted file mode 100644 index c2705fcbdc..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.context.event.EventListener; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.queue.TbPackCallback; -import org.thingsboard.server.service.queue.TbPackProcessingContext; - -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Slf4j -public abstract class AbstractConsumerService implements ApplicationListener { - - protected volatile ExecutorService consumersExecutor; - protected volatile ExecutorService notificationsConsumerExecutor; - protected volatile boolean stopped = false; - - protected final ActorSystemContext actorContext; - protected final DataDecodingEncodingService encodingService; - - protected final TbQueueConsumer> nfConsumer; - - public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbQueueConsumer> nfConsumer) { - this.actorContext = actorContext; - this.encodingService = encodingService; - this.nfConsumer = nfConsumer; - } - - public void init(String mainConsumerThreadName, String nfConsumerThreadName) { - this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName(mainConsumerThreadName)); - this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(nfConsumerThreadName)); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent event) { - log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); - this.nfConsumer.subscribe(); - launchNotificationsConsumer(); - launchMainConsumers(); - } - - protected abstract ServiceType getServiceType(); - - protected abstract void launchMainConsumers(); - - protected abstract void stopMainConsumers(); - - protected abstract long getNotificationPollDuration(); - - protected abstract long getNotificationPackProcessingTimeout(); - - protected void launchNotificationsConsumer() { - notificationsConsumerExecutor.submit(() -> { - while (!stopped) { - try { - List> msgs = nfConsumer.poll(getNotificationPollDuration()); - if (msgs.isEmpty()) { - continue; - } - ConcurrentMap> pendingMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - TbPackProcessingContext> ctx = new TbPackProcessingContext<>( - processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - pendingMap.forEach((id, msg) -> { - log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); - TbCallback callback = new TbPackCallback<>(id, ctx); - try { - handleNotification(id, msg, callback); - } catch (Throwable e) { - log.warn("[{}] Failed to process notification: {}", id, msg, e); - callback.onFailure(e); - } - }); - if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { - ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); - ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); - } - nfConsumer.commit(); - } catch (Exception e) { - if (!stopped) { - log.warn("Failed to obtain notifications from queue.", e); - try { - Thread.sleep(getNotificationPollDuration()); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new notifications", e2); - } - } - } - } - log.info("TB Notifications Consumer stopped."); - }); - } - - protected abstract void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception; - - @PreDestroy - public void destroy() { - stopped = true; - - stopMainConsumers(); - - if (nfConsumer != null) { - nfConsumer.unsubscribe(); - } - - if (consumersExecutor != null) { - consumersExecutor.shutdownNow(); - } - if (notificationsConsumerExecutor != null) { - notificationsConsumerExecutor.shutdownNow(); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java deleted file mode 100644 index bef733ec22..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.stream.Collectors; - -public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngineSubmitStrategy { - - protected final String queueName; - protected List orderedMsgList; - private volatile boolean stopped; - - public AbstractTbRuleEngineSubmitStrategy(String queueName) { - this.queueName = queueName; - } - - protected abstract void doOnSuccess(UUID id); - - @Override - public void init(List> msgs) { - orderedMsgList = msgs.stream().map(msg -> new IdMsgPair(UUID.randomUUID(), msg)).collect(Collectors.toList()); - } - - @Override - public ConcurrentMap> getPendingMap() { - return orderedMsgList.stream().collect(Collectors.toConcurrentMap(pair -> pair.uuid, pair -> pair.msg)); - } - - @Override - public void update(ConcurrentMap> reprocessMap) { - List newOrderedMsgList = new ArrayList<>(reprocessMap.size()); - for (IdMsgPair pair : orderedMsgList) { - if (reprocessMap.containsKey(pair.uuid)) { - newOrderedMsgList.add(pair); - } - } - orderedMsgList = newOrderedMsgList; - } - - @Override - public void onSuccess(UUID id) { - if (!stopped) { - doOnSuccess(id); - } - } - - @Override - public void stop() { - stopped = true; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java deleted file mode 100644 index d0b1f7f99a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; - -@Slf4j -public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { - - private final int batchSize; - private final AtomicInteger packIdx = new AtomicInteger(0); - private final Map> pendingPack = new LinkedHashMap<>(); - private volatile BiConsumer> msgConsumer; - - public BatchTbRuleEngineSubmitStrategy(String queueName, int batchSize) { - super(queueName); - this.batchSize = batchSize; - } - - @Override - public void submitAttempt(BiConsumer> msgConsumer) { - this.msgConsumer = msgConsumer; - submitNext(); - } - - @Override - public void update(ConcurrentMap> reprocessMap) { - super.update(reprocessMap); - packIdx.set(0); - } - - @Override - protected void doOnSuccess(UUID id) { - boolean endOfPendingPack; - synchronized (pendingPack) { - TbProtoQueueMsg msg = pendingPack.remove(id); - endOfPendingPack = msg != null && pendingPack.isEmpty(); - } - if (endOfPendingPack) { - packIdx.incrementAndGet(); - submitNext(); - } - } - - private void submitNext() { - int listSize = orderedMsgList.size(); - int startIdx = Math.min(packIdx.get() * batchSize, listSize); - int endIdx = Math.min(startIdx + batchSize, listSize); - synchronized (pendingPack) { - pendingPack.clear(); - for (int i = startIdx; i < endIdx; i++) { - IdMsgPair pair = orderedMsgList.get(i); - pendingPack.put(pair.uuid, pair.msg); - } - } - int submitSize = pendingPack.size(); - if (log.isInfoEnabled() && submitSize > 0) { - log.info("[{}] submitting [{}] messages to rule engine", queueName, submitSize); - } - pendingPack.forEach(msgConsumer); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java deleted file mode 100644 index ffd1dd49d1..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Slf4j -public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { - - public BurstTbRuleEngineSubmitStrategy(String queueName) { - super(queueName); - } - - @Override - public void submitAttempt(BiConsumer> msgConsumer) { - if (log.isInfoEnabled()) { - log.info("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); - } - orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg)); - } - - @Override - protected void doOnSuccess(UUID id) { - - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java deleted file mode 100644 index ae5993cb1c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Queue; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; -import java.util.stream.Collectors; - -@Slf4j -public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { - - private volatile BiConsumer> msgConsumer; - private volatile ConcurrentMap msgToEntityIdMap = new ConcurrentHashMap<>(); - private volatile ConcurrentMap> entityIdToListMap = new ConcurrentHashMap<>(); - - public SequentialByEntityIdTbRuleEngineSubmitStrategy(String queueName) { - super(queueName); - } - - @Override - public void init(List> msgs) { - super.init(msgs); - initMaps(); - } - - @Override - public void submitAttempt(BiConsumer> msgConsumer) { - this.msgConsumer = msgConsumer; - entityIdToListMap.forEach((entityId, queue) -> { - IdMsgPair msg = queue.peek(); - if (msg != null) { - msgConsumer.accept(msg.uuid, msg.msg); - } - }); - } - - @Override - public void update(ConcurrentMap> reprocessMap) { - super.update(reprocessMap); - initMaps(); - } - - @Override - protected void doOnSuccess(UUID id) { - EntityId entityId = msgToEntityIdMap.get(id); - if (entityId != null) { - Queue queue = entityIdToListMap.get(entityId); - if (queue != null) { - IdMsgPair next = null; - synchronized (queue) { - IdMsgPair expected = queue.peek(); - if (expected != null && expected.uuid.equals(id)) { - queue.poll(); - next = queue.peek(); - } - } - if (next != null) { - msgConsumer.accept(next.uuid, next.msg); - } - } - } - } - - private void initMaps() { - msgToEntityIdMap.clear(); - entityIdToListMap.clear(); - for (IdMsgPair pair : orderedMsgList) { - EntityId entityId = getEntityId(pair.msg.getValue()); - if (entityId != null) { - msgToEntityIdMap.put(pair.uuid, entityId); - entityIdToListMap.computeIfAbsent(entityId, id -> new LinkedList<>()).add(pair); - } - } - } - - protected abstract EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java deleted file mode 100644 index cd8a97e82c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.UUID; - -@Slf4j -public class SequentialByOriginatorIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { - - public SequentialByOriginatorIdTbRuleEngineSubmitStrategy(String queueName) { - super(queueName); - } - - @Override - protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { - try { - MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg()); - return EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - } catch (InvalidProtocolBufferException e) { - log.warn("[{}] Failed to parse TbMsg: {}", queueName, msg); - return null; - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java deleted file mode 100644 index ef45b983fc..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; - -@Slf4j -public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { - - private final AtomicInteger msgIdx = new AtomicInteger(0); - private volatile BiConsumer> msgConsumer; - private volatile UUID expectedMsgId; - - public SequentialTbRuleEngineSubmitStrategy(String queueName) { - super(queueName); - } - - @Override - public void submitAttempt(BiConsumer> msgConsumer) { - this.msgConsumer = msgConsumer; - msgIdx.set(0); - submitNext(); - } - - @Override - public void update(ConcurrentMap> reprocessMap) { - super.update(reprocessMap); - } - - @Override - protected void doOnSuccess(UUID id) { - if (expectedMsgId.equals(id)) { - msgIdx.incrementAndGet(); - submitNext(); - } - } - - private void submitNext() { - int listSize = orderedMsgList.size(); - int idx = msgIdx.get(); - if (idx < listSize) { - IdMsgPair pair = orderedMsgList.get(idx); - expectedMsgId = pair.uuid; - if (log.isInfoEnabled()) { - log.info("[{}] submitting [{}] message to rule engine", queueName, pair.msg); - } - msgConsumer.accept(pair.uuid, pair.msg); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java deleted file mode 100644 index e818ffb9d7..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.Getter; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; - -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; - -public class TbRuleEngineProcessingResult { - - @Getter - private final boolean success; - @Getter - private final boolean timeout; - @Getter - private final TbMsgPackProcessingContext ctx; - - public TbRuleEngineProcessingResult(boolean timeout, TbMsgPackProcessingContext ctx) { - this.timeout = timeout; - this.ctx = ctx; - this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty(); - } - - public ConcurrentMap> getPendingMap() { - return ctx.getPendingMap(); - } - - public ConcurrentMap> getSuccessMap() { - return ctx.getSuccessMap(); - } - - public ConcurrentMap> getFailedMap() { - return ctx.getFailedMap(); - } - - public ConcurrentMap getExceptionsMap() { - return ctx.getExceptionsMap(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java deleted file mode 100644 index cfba85f5dc..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -public interface TbRuleEngineProcessingStrategy { - - TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java deleted file mode 100644 index bbf283e962..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration; - -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; - -@Component -@Slf4j -public class TbRuleEngineProcessingStrategyFactory { - - public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) { - switch (configuration.getType()) { - case "SKIP_ALL_FAILURES": - return new SkipStrategy(name); - case "RETRY_ALL": - return new RetryStrategy(name, true, true, true, configuration); - case "RETRY_FAILED": - return new RetryStrategy(name, false, true, false, configuration); - case "RETRY_TIMED_OUT": - return new RetryStrategy(name, false, false, true, configuration); - case "RETRY_FAILED_AND_TIMED_OUT": - return new RetryStrategy(name, false, true, true, configuration); - default: - throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); - } - } - - private static class RetryStrategy implements TbRuleEngineProcessingStrategy { - private final String queueName; - private final boolean retrySuccessful; - private final boolean retryFailed; - private final boolean retryTimeout; - private final int maxRetries; - private final double maxAllowedFailurePercentage; - private final long pauseBetweenRetries; - - private int initialTotalCount; - private int retryCount; - - public RetryStrategy(String queueName, boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, TbRuleEngineQueueAckStrategyConfiguration configuration) { - this.queueName = queueName; - this.retrySuccessful = retrySuccessful; - this.retryFailed = retryFailed; - this.retryTimeout = retryTimeout; - this.maxRetries = configuration.getRetries(); - this.maxAllowedFailurePercentage = configuration.getFailurePercentage(); - this.pauseBetweenRetries = configuration.getPauseBetweenRetries(); - } - - @Override - public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { - if (result.isSuccess()) { - return new TbRuleEngineProcessingDecision(true, null); - } else { - if (retryCount == 0) { - initialTotalCount = result.getPendingMap().size() + result.getFailedMap().size() + result.getSuccessMap().size(); - } - retryCount++; - double failedCount = result.getFailedMap().size() + result.getPendingMap().size(); - if (maxRetries > 0 && retryCount > maxRetries) { - log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); - return new TbRuleEngineProcessingDecision(true, null); - } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { - log.info("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); - return new TbRuleEngineProcessingDecision(true, null); - } else { - ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); - if (retryFailed) { - result.getFailedMap().forEach(toReprocess::put); - } - if (retryTimeout) { - result.getPendingMap().forEach(toReprocess::put); - } - if (retrySuccessful) { - result.getSuccessMap().forEach(toReprocess::put); - } - log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); - if (log.isTraceEnabled()) { - toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); - } - if (pauseBetweenRetries > 0) { - try { - Thread.sleep(TimeUnit.SECONDS.toMillis(pauseBetweenRetries)); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - return new TbRuleEngineProcessingDecision(false, toReprocess); - } - } - } - } - - private static class SkipStrategy implements TbRuleEngineProcessingStrategy { - - private final String queueName; - - public SkipStrategy(String name) { - this.queueName = name; - } - - @Override - public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { - if (!result.isSuccess()) { - log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); - } - if (log.isTraceEnabled()) { - result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); - } - if (log.isTraceEnabled()) { - result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); - } - return new TbRuleEngineProcessingDecision(true, null); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java deleted file mode 100644 index 7b22da97db..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.function.BiConsumer; - -public interface TbRuleEngineSubmitStrategy { - - void init(List> msgs); - - ConcurrentMap> getPendingMap(); - - void submitAttempt(BiConsumer> msgConsumer); - - void update(ConcurrentMap> reprocessMap); - - void onSuccess(UUID id); - - void stop(); -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java deleted file mode 100644 index 90eb8561a0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue.processing; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueSubmitStrategyConfiguration; - -@Component -@Slf4j -public class TbRuleEngineSubmitStrategyFactory { - - public TbRuleEngineSubmitStrategy newInstance(String name, TbRuleEngineQueueSubmitStrategyConfiguration configuration) { - switch (configuration.getType()) { - case "BURST": - return new BurstTbRuleEngineSubmitStrategy(name); - case "BATCH": - return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize()); - case "SEQUENTIAL_BY_ORIGINATOR": - return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name); - case "SEQUENTIAL_BY_TENANT": - return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name); - case "SEQUENTIAL": - return new SequentialTbRuleEngineSubmitStrategy(name); - default: - throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java new file mode 100644 index 0000000000..5d303b54ab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java @@ -0,0 +1,222 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import com.datastax.driver.core.utils.UUIDs; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 27.03.18. + */ +@Service +@Slf4j +public class DefaultDeviceRpcService implements DeviceRpcService { + + private static final ObjectMapper json = new ObjectMapper(); + + @Autowired + private ClusterRoutingService routingService; + + @Autowired + private ClusterRpcService rpcService; + + @Autowired + private DeviceService deviceService; + + @Autowired + @Lazy + private ActorService actorService; + + private ScheduledExecutorService rpcCallBackExecutor; + + private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); + private final ConcurrentMap> localToDeviceRpcRequests = new ConcurrentHashMap<>(); + + @PostConstruct + public void initExecutor() { + rpcCallBackExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rpc-callback")); + } + + @PreDestroy + public void shutdownExecutor() { + if (rpcCallBackExecutor != null) { + rpcCallBackExecutor.shutdownNow(); + } + } + + @Override + public void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToRuleEngineRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToRuleEngine(request); + scheduleTimeout(request, requestId, localToRuleEngineRpcRequests); + } + + @Override + public void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), requestOriginAddress); + if (routingService.getCurrentServer().equals(requestOriginAddress)) { + UUID requestId = response.getId(); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } else { + ClusterAPIProtos.FromDeviceRPCResponseProto.Builder builder = ClusterAPIProtos.FromDeviceRPCResponseProto.newBuilder(); + builder.setRequestIdMSB(response.getId().getMostSignificantBits()); + builder.setRequestIdLSB(response.getId().getLeastSignificantBits()); + response.getResponse().ifPresent(builder::setResponse); + if (response.getError().isPresent()) { + builder.setError(response.getError().get().ordinal()); + } else { + builder.setError(-1); + } + rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); + } + } + + @Override + public void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToDeviceRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToDevice(request); + scheduleTimeout(request, requestId, localToDeviceRpcRequests); + } + + @Override + public void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); + UUID requestId = response.getId(); + Consumer consumer = localToDeviceRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + @Override + public void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.FromDeviceRPCResponseProto proto; + try { + proto = ClusterAPIProtos.FromDeviceRPCResponseProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; + FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), proto.getResponse(), error); + processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); + } + + @Override + public void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body) { + ToServerRpcResponseActorMsg rpcMsg = new ToServerRpcResponseActorMsg(tenantId, deviceId, new ToServerRpcResponseMsg(requestId, body)); + forward(deviceId, rpcMsg); + } + + private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { + ObjectNode entityNode = json.createObjectNode(); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("requestUUID", msg.getId().toString()); + metaData.putValue("originHost", routingService.getCurrentServer().getHost()); + metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); + metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); + metaData.putValue("oneway", Boolean.toString(msg.isOneway())); + + Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); + if (device != null) { + metaData.putValue("deviceName", device.getName()); + metaData.putValue("deviceType", device.getType()); + } + + entityNode.put("method", msg.getBody().getMethod()); + entityNode.put("params", msg.getBody().getParams()); + + try { + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON + , json.writeValueAsString(entityNode) + , null, null, 0L); + actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new ServiceToRuleEngineMsg(msg.getTenantId(), tbMsg))); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { + ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); + log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); + forward(msg.getDeviceId(), rpcMsg); + } + + private void forward(DeviceId deviceId, T msg) { + actorService.onMsg(new SendToClusterMsg(deviceId, msg)); + } + + private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId, ConcurrentMap> requestsMap) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); + rpcCallBackExecutor.schedule(() -> { + log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); + Consumer consumer = requestsMap.remove(requestId); + if (consumer != null) { + consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); + } + }, timeout, TimeUnit.MILLISECONDS); + } + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java deleted file mode 100644 index f6e5eb8b43..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java +++ /dev/null @@ -1,199 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import akka.actor.ActorRef; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.queue.TbClusterService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 27.03.18. - */ -@Service -@Slf4j -@TbCoreComponent -public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { - - private static final ObjectMapper json = new ObjectMapper(); - - private final DeviceService deviceService; - private final TbClusterService clusterService; - private final TbServiceInfoProvider serviceInfoProvider; - private final ActorSystemContext actorContext; - - private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); - private final ConcurrentMap localToDeviceRpcRequests = new ConcurrentHashMap<>(); - - private Optional tbRuleEngineRpcService; - private ScheduledExecutorService scheduler; - private String serviceId; - - public DefaultTbCoreDeviceRpcService(DeviceService deviceService, TbClusterService clusterService, TbServiceInfoProvider serviceInfoProvider, - ActorSystemContext actorContext) { - this.deviceService = deviceService; - this.clusterService = clusterService; - this.serviceInfoProvider = serviceInfoProvider; - this.actorContext = actorContext; - } - - @Autowired(required = false) - public void setTbRuleEngineRpcService(Optional tbRuleEngineRpcService) { - this.tbRuleEngineRpcService = tbRuleEngineRpcService; - } - - @PostConstruct - public void initExecutor() { - scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-core-rpc-scheduler")); - serviceId = serviceInfoProvider.getServiceId(); - } - - @PreDestroy - public void shutdownExecutor() { - if (scheduler != null) { - scheduler.shutdownNow(); - } - } - - @Override - public void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToRuleEngineRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToRuleEngine(request); - scheduleToRuleEngineTimeout(request, requestId); - } - - @Override - public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), response); - UUID requestId = response.getId(); - Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } - - @Override - public void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg rpcMsg) { - ToDeviceRpcRequest request = rpcMsg.getMsg(); - log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToDeviceRpcRequests.put(requestId, rpcMsg); - actorContext.tell(rpcMsg, ActorRef.noSender()); - scheduleToDeviceTimeout(request, requestId); - } - - @Override - public void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); - UUID requestId = response.getId(); - ToDeviceRpcRequestActorMsg request = localToDeviceRpcRequests.remove(requestId); - if (request != null) { - sendRpcResponseToTbRuleEngine(request.getServiceId(), response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } - - private void sendRpcResponseToTbRuleEngine(String originServiceId, FromDeviceRpcResponse response) { - if (serviceId.equals(originServiceId)) { - if (tbRuleEngineRpcService.isPresent()) { - tbRuleEngineRpcService.get().processRpcResponseFromDevice(response); - } else { - log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); - } - } else { - clusterService.pushNotificationToRuleEngine(originServiceId, response, null); - } - } - - private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { - ObjectNode entityNode = json.createObjectNode(); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("requestUUID", msg.getId().toString()); - metaData.putValue("originServiceId", serviceId); - metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); - metaData.putValue("oneway", Boolean.toString(msg.isOneway())); - - Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); - if (device != null) { - metaData.putValue("deviceName", device.getName()); - metaData.putValue("deviceType", device.getType()); - } - - entityNode.put("method", msg.getBody().getMethod()); - entityNode.put("params", msg.getBody().getParams()); - - try { - TbMsg tbMsg = TbMsg.newMsg(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); - clusterService.pushMsgToRuleEngine(msg.getTenantId(), msg.getDeviceId(), tbMsg, null); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) { - long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing to rule engine request.", requestId); - scheduler.schedule(() -> { - log.trace("[{}] timeout for processing to rule engine request.", requestId); - Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); - } - }, timeout, TimeUnit.MILLISECONDS); - } - - private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) { - long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing to device request.", requestId); - scheduler.schedule(() -> { - log.trace("[{}] timeout for to device request.", requestId); - localToDeviceRpcRequests.remove(requestId); - }, timeout, TimeUnit.MILLISECONDS); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java deleted file mode 100644 index 0ec730b7dc..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; -import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.util.TbRuleEngineComponent; -import org.thingsboard.server.service.queue.TbClusterService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -@Service -@TbRuleEngineComponent -@Slf4j -public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcService { - - private final PartitionService partitionService; - private final TbClusterService clusterService; - private final TbServiceInfoProvider serviceInfoProvider; - - - private final ConcurrentMap> toDeviceRpcRequests = new ConcurrentHashMap<>(); - - private Optional tbCoreRpcService; - private ScheduledExecutorService scheduler; - private String serviceId; - - public DefaultTbRuleEngineRpcService(PartitionService partitionService, - TbClusterService clusterService, - TbServiceInfoProvider serviceInfoProvider) { - this.partitionService = partitionService; - this.clusterService = clusterService; - this.serviceInfoProvider = serviceInfoProvider; - } - - @Autowired(required = false) - public void setTbCoreRpcService(Optional tbCoreRpcService) { - this.tbCoreRpcService = tbCoreRpcService; - } - - @PostConstruct - public void initExecutor() { - scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rule-engine-rpc-scheduler")); - serviceId = serviceInfoProvider.getServiceId(); - } - - @PreDestroy - public void shutdownExecutor() { - if (scheduler != null) { - scheduler.shutdownNow(); - } - } - - @Override - public void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body) { - TransportProtos.ToServerRpcResponseMsg responseMsg = TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(requestId) - .setPayload(body).build(); - TransportProtos.ToTransportMsg msg = TransportProtos.ToTransportMsg.newBuilder() - .setSessionIdMSB(sessionId.getMostSignificantBits()) - .setSessionIdLSB(sessionId.getLeastSignificantBits()) - .setToServerResponse(responseMsg) - .build(); - clusterService.pushNotificationToTransport(serviceId, msg, null); - } - - @Override - public void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest src, Consumer consumer) { - ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), src.getTenantId(), src.getDeviceId(), - src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); - forwardRpcRequestToDeviceActor(request, response -> { - if (src.isRestApiCall()) { - sendRpcResponseToTbCore(src.getOriginServiceId(), response); - } - consumer.accept(RuleEngineDeviceRpcResponse.builder() - .deviceId(src.getDeviceId()) - .requestId(src.getRequestId()) - .error(response.getError()) - .response(response.getResponse()) - .build()); - }); - } - - @Override - public void processRpcResponseFromDevice(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from Core RPC Service", response.getId()); - UUID requestId = response.getId(); - Consumer consumer = toDeviceRpcRequests.remove(requestId); - if (consumer != null) { - scheduler.submit(() -> consumer.accept(response)); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } - - private void forwardRpcRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - toDeviceRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToDevice(request); - scheduleTimeout(request, requestId); - } - - private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); - ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(serviceId, msg); - if (tpi.isMyPartition()) { - log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); - if (tbCoreRpcService.isPresent()) { - tbCoreRpcService.get().forwardRpcRequestToDeviceActor(rpcMsg); - } else { - log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); - } - } else { - log.trace("[{}] Forwarding msg {} to queue actor!", msg.getDeviceId(), msg); - clusterService.pushMsgToCore(rpcMsg, null); - } - } - - private void sendRpcResponseToTbCore(String originServiceId, FromDeviceRpcResponse response) { - if (serviceId.equals(originServiceId)) { - if (tbCoreRpcService.isPresent()) { - tbCoreRpcService.get().processRpcResponseFromRuleEngine(response); - } else { - log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); - } - } else { - clusterService.pushNotificationToCore(originServiceId, response, null); - } - } - - private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId) { - long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); - scheduler.schedule(() -> { - log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); - Consumer consumer = toDeviceRpcRequests.remove(requestId); - if (consumer != null) { - scheduler.submit(() -> consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT))); - } - }, timeout, TimeUnit.MILLISECONDS); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java new file mode 100644 index 0000000000..feb5d58c3d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; + +import java.util.function.Consumer; + +/** + * Created by ashvayka on 16.04.18. + */ +public interface DeviceRpcService { + + void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer); + + void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response); + + void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer); + + void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response); + + void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data); + + void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body); +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java index c1e5e2e038..78ea0c5660 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java @@ -19,6 +19,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.common.msg.cluster.ServerAddress; import java.util.Optional; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java deleted file mode 100644 index 896745c48e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; - -import java.util.function.Consumer; - -/** - * Handles REST API calls that contain RPC requests to Device. - */ -public interface TbCoreDeviceRpcService { - - /** - * Handles REST API calls that contain RPC requests to Device and pushes them to Rule Engine. - * Schedules the timeout for the RPC call based on the {@link ToDeviceRpcRequest} - * - * @param request the RPC request - * @param responseConsumer the consumer of the RPC response - */ - void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer); - - /** - * Handles the RPC response from the Rule Engine. - * - * @param response the RPC response - */ - void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response); - - /** - * Forwards the RPC request from Rule Engine to Device Actor - * - * @param request the RPC request message - */ - void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg request); - - /** - * Handles the RPC response from the Device Actor (Transport). - * - * @param response the RPC response - */ - void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java index 3cfe9bc1f4..ba537aed25 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java @@ -22,8 +22,11 @@ import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import java.util.Optional; + /** * Created by ashvayka on 16.04.18. */ @@ -32,7 +35,7 @@ import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; public class ToDeviceRpcRequestActorMsg implements ToDeviceActorNotificationMsg { @Getter - private final String serviceId; + private final ServerAddress serverAddress; @Getter private final ToDeviceRpcRequest msg; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java similarity index 50% rename from common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java rename to application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java index f4bfefd878..570112dab3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java @@ -13,42 +13,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.queue; +package org.thingsboard.server.service.rpc; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; -import java.util.Objects; - +/** + * Created by ashvayka on 16.04.18. + */ @ToString -public class ServiceQueueKey { - @Getter - private final ServiceQueue serviceQueue; +@RequiredArgsConstructor +public class ToServerRpcResponseActorMsg implements ToDeviceActorNotificationMsg { @Getter private final TenantId tenantId; - public ServiceQueueKey(ServiceQueue serviceQueue, TenantId tenantId) { - this.serviceQueue = serviceQueue; - this.tenantId = tenantId; - } + @Getter + private final DeviceId deviceId; - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ServiceQueueKey that = (ServiceQueueKey) o; - return serviceQueue.equals(that.serviceQueue) && - Objects.equals(tenantId, that.tenantId); - } + @Getter + private final ToServerRpcResponseMsg msg; @Override - public int hashCode() { - return Objects.hash(serviceQueue, tenantId); - } - - public ServiceType getServiceType() { - return serviceQueue.getType(); + public MsgType getMsgType() { + return MsgType.SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG; } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index daa74d013f..31827f542e 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractJsInvokeService implements JsInvokeService { protected Map scriptIdToNameMap = new ConcurrentHashMap<>(); - protected Map blackListedFunctions = new ConcurrentHashMap<>(); + protected Map blackListedFunctions = new ConcurrentHashMap<>(); @Override public ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames) { @@ -78,53 +78,25 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract int getMaxErrors(); - protected abstract long getMaxBlacklistDuration(); - protected void onScriptExecutionError(UUID scriptId) { - blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); + blackListedFunctions.computeIfAbsent(scriptId, key -> new AtomicInteger(0)).incrementAndGet(); } private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { - if (scriptType == JsScriptType.RULE_NODE_SCRIPT) { - return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); + switch (scriptType) { + case RULE_NODE_SCRIPT: + return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); + default: + throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); } - throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); } private boolean isBlackListed(UUID scriptId) { - BlackListInfo errorCount = blackListedFunctions.get(scriptId); - if (errorCount != null) { - if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { - blackListedFunctions.remove(scriptId); - return false; - } else { - return errorCount.get() >= getMaxErrors(); - } + if (blackListedFunctions.containsKey(scriptId)) { + AtomicInteger errorCount = blackListedFunctions.get(scriptId); + return errorCount.get() >= getMaxErrors(); } else { return false; } } - - private class BlackListInfo { - private final AtomicInteger counter; - private long expirationTime; - - private BlackListInfo() { - this.counter = new AtomicInteger(0); - } - - public int get() { - return counter.get(); - } - - public int incrementAndGet() { - int result = counter.incrementAndGet(); - expirationTime = System.currentTimeMillis() + getMaxBlacklistDuration(); - return result; - } - - public long getExpirationTime() { - return expirationTime; - } - } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index a9acfa4f42..49a6304ebb 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -55,8 +55,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer private final AtomicInteger jsEvalMsgs = new AtomicInteger(0); private final AtomicInteger jsFailedMsgs = new AtomicInteger(0); private final AtomicInteger jsTimeoutMsgs = new AtomicInteger(0); - private final FutureCallback evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); - private final FutureCallback invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback evalCallback = new JsStatCallback(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback invokeCallback = new JsStatCallback(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); @Autowired @Getter diff --git a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java index 66a14cc827..909565609c 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java @@ -20,8 +20,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import java.util.concurrent.TimeUnit; - @Slf4j @ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "local", matchIfMissing = true) @Service @@ -39,9 +37,6 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { @Value("${js.local.max_errors}") private int maxErrors; - @Value("${js.local.max_black_list_duration_sec:60}") - private int maxBlackListDurationSec; - @Override protected boolean useJsSandbox() { return useJsSandbox; @@ -61,9 +56,4 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } - - @Override - protected long getMaxBlacklistDuration() { - return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 8d1f9d662a..1a218b119b 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -23,13 +23,15 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaRequestTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -37,25 +39,42 @@ import javax.annotation.PreDestroy; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -@ConditionalOnExpression("'${js.evaluator:null}'=='remote' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine')") +@ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "remote", matchIfMissing = true) @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { + @Autowired + private TbNodeIdProvider nodeIdProvider; + + @Autowired + private TbKafkaSettings kafkaSettings; + + @Value("${js.remote.request_topic}") + private String requestTopic; + + @Value("${js.remote.response_topic_prefix}") + private String responseTopicPrefix; + + @Value("${js.remote.max_pending_requests}") + private long maxPendingRequests; + @Value("${js.remote.max_requests_timeout}") private long maxRequestsTimeout; + @Value("${js.remote.response_poll_interval}") + private int responsePollDuration; + + @Value("${js.remote.response_auto_commit_interval}") + private int autoCommitInterval; + @Getter @Value("${js.remote.max_errors}") private int maxErrors; - @Value("${js.remote.max_black_list_duration_sec:60}") - private int maxBlackListDurationSec; - @Value("${js.remote.stats.enabled:false}") private boolean statsEnabled; @@ -80,20 +99,42 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } - @Autowired - private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; - + private TbKafkaRequestTemplate kafkaTemplate; private Map scriptIdToBodysMap = new ConcurrentHashMap<>(); @PostConstruct public void init() { - requestTemplate.init(); + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(requestTopic); + requestBuilder.encoder(new RemoteJsRequestEncoder()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(responseTopicPrefix + "." + nodeIdProvider.getNodeId()); + responseBuilder.clientId("js-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + responseBuilder.autoCommitIntervalMs(autoCommitInterval); + responseBuilder.decoder(new RemoteJsResponseDecoder()); + responseBuilder.requestIdExtractor((response) -> new UUID(response.getRequestIdMSB(), response.getRequestIdLSB())); + + TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder + builder = TbKafkaRequestTemplate.builder(); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(maxPendingRequests); + builder.maxRequestTimeout(maxRequestsTimeout); + builder.pollInterval(responsePollDuration); + kafkaTemplate = builder.build(); + kafkaTemplate.init(); } @PreDestroy public void destroy() { - if (requestTemplate != null) { - requestTemplate.stop(); + if (kafkaTemplate != null) { + kafkaTemplate.stop(); } } @@ -110,12 +151,11 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .build(); log.trace("Post compile request for scriptId [{}]", scriptId); - ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); - + ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback>() { + Futures.addCallback(future, new FutureCallback() { @Override - public void onSuccess(@Nullable TbProtoQueueMsg result) { + public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { kafkaEvalMsgs.incrementAndGet(); } @@ -128,7 +168,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }, MoreExecutors.directExecutor()); return Futures.transform(future, response -> { - JsInvokeProtos.JsCompileResponse compilationResult = response.getValue().getCompileResponse(); + JsInvokeProtos.JsCompileResponse compilationResult = response.getCompileResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); if (compilationResult.getSuccess()) { scriptIdToNameMap.put(scriptId, functionName); @@ -162,17 +202,16 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setInvokeRequest(jsRequestBuilder.build()) .build(); - ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback>() { + Futures.addCallback(future, new FutureCallback() { @Override - public void onSuccess(@Nullable TbProtoQueueMsg result) { + public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { kafkaInvokeMsgs.incrementAndGet(); } @Override public void onFailure(Throwable t) { - onScriptExecutionError(scriptId); if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { kafkaTimeoutMsgs.incrementAndGet(); } @@ -180,11 +219,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }, MoreExecutors.directExecutor()); return Futures.transform(future, response -> { - JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse(); + JsInvokeProtos.JsInvokeResponse invokeResult = response.getInvokeResponse(); if (invokeResult.getSuccess()) { return invokeResult.getResult(); } else { - onScriptExecutionError(scriptId); log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); throw new RuntimeException(invokeResult.getErrorDetails()); } @@ -202,8 +240,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setReleaseRequest(jsRequest) .build(); - ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); - JsInvokeProtos.RemoteJsResponse response = future.get().getValue(); + ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); + JsInvokeProtos.RemoteJsResponse response = future.get(); JsInvokeProtos.JsReleaseResponse compilationResult = response.getReleaseResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); @@ -214,9 +252,4 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } - @Override - protected long getMaxBlacklistDuration() { - return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java index 51ecba27b1..d07a2490ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java @@ -17,20 +17,19 @@ package org.thingsboard.server.service.script; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.queue.kafka.TbKafkaEncoder; +import org.thingsboard.server.kafka.TbKafkaEncoder; import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsRequestEncoder implements TbKafkaEncoder> { +public class RemoteJsRequestEncoder implements TbKafkaEncoder { @Override - public byte[] encode(TbProtoQueueMsg value) { + public byte[] encode(JsInvokeProtos.RemoteJsRequest value) { try { - return JsonFormat.printer().print(value.getValue()).getBytes(StandardCharsets.UTF_8); + return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java index 621d2b05d7..7a3876fb91 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java @@ -16,10 +16,8 @@ package org.thingsboard.server.service.script; import com.google.protobuf.util.JsonFormat; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.queue.kafka.TbKafkaDecoder; +import org.thingsboard.server.kafka.TbKafkaDecoder; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -27,12 +25,12 @@ import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsResponseDecoder implements TbKafkaDecoder> { +public class RemoteJsResponseDecoder implements TbKafkaDecoder { @Override - public TbProtoQueueMsg decode(TbQueueMsg msg) throws IOException { + public JsInvokeProtos.RemoteJsResponse decode(byte[] data) throws IOException { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + JsonFormat.parser().ignoringUnknownFields().merge(new String(data, StandardCharsets.UTF_8), builder); + return builder.build(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 110f410fad..ef5d4716cb 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -95,7 +95,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S String newData = data != null ? data : msg.getData(); TbMsgMetaData newMetadata = metadata != null ? new TbMsgMetaData(metadata) : msg.getMetaData().copy(); String newMessageType = !StringUtils.isEmpty(messageType) ? messageType : msg.getType(); - return TbMsg.transformMsg(msg, newMessageType, msg.getOriginator(), newMetadata, newData); + return new TbMsg(msg.getId(), newMessageType, msg.getOriginator(), newMetadata, newData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); } catch (Throwable th) { th.printStackTrace(); throw new RuntimeException("Failed to unbind message data from javascript result", th); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index 572c3a6ed0..e359befb30 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -46,7 +46,6 @@ import ua_parser.Client; import java.util.UUID; - @Component @Slf4j public class RestAuthenticationProvider implements AuthenticationProvider { diff --git a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java index c13cb5529a..c58a664790 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.security.device; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsFilter; @@ -23,21 +24,16 @@ import org.thingsboard.server.common.transport.auth.DeviceAuthResult; import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.queue.util.TbCoreComponent; @Service -@TbCoreComponent @Slf4j public class DefaultDeviceAuthService implements DeviceAuthService { - private final DeviceService deviceService; + @Autowired + DeviceService deviceService; - private final DeviceCredentialsService deviceCredentialsService; - - public DefaultDeviceAuthService(DeviceService deviceService, DeviceCredentialsService deviceCredentialsService) { - this.deviceService = deviceService; - this.deviceCredentialsService = deviceCredentialsService; - } + @Autowired + DeviceCredentialsService deviceCredentialsService; @Override public DeviceAuthResult process(DeviceCredentialsFilter credentialsFilter) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java index 5412ed6def..40d19da764 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.service.security.permission; +import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; public interface AccessControlService { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java index a8c5dc13ae..9777af7b8e 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java @@ -16,13 +16,20 @@ package org.thingsboard.server.service.security.permission; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.*; diff --git a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java index 1b9d566206..952c63c9b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java +++ b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java @@ -21,9 +21,9 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; -import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.Collections; +import java.util.UUID; import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; @@ -31,7 +31,6 @@ import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; * Created by ashvayka on 29.10.18. */ @Service -@TbCoreComponent @Slf4j public class DefaultDeviceSessionCacheService implements DeviceSessionCacheService { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 24eea18b53..99a24cfb12 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.state; +import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.util.concurrent.FutureCallback; @@ -22,13 +23,16 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; @@ -45,18 +49,16 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -67,6 +69,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -86,8 +89,8 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; * Created by ashvayka on 01.05.18. */ @Service -@TbCoreComponent @Slf4j +//TODO: refactor to use page links as cursor and not fetch all public class DefaultDeviceStateService implements DeviceStateService { private static final ObjectMapper json = new ObjectMapper(); @@ -101,15 +104,31 @@ public class DefaultDeviceStateService implements DeviceStateService { public static final List PERSISTENT_ATTRIBUTES = Arrays.asList(ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT); - private final TenantService tenantService; - private final DeviceService deviceService; - private final AttributesService attributesService; - private final TimeseriesService tsService; - private final TbClusterService clusterService; - private final PartitionService partitionService; + @Autowired + private TenantService tenantService; + + @Autowired + private DeviceService deviceService; + @Autowired + private AttributesService attributesService; + + @Autowired + private TimeseriesService tsService; + + @Autowired + @Lazy + private ActorService actorService; + + @Autowired private TelemetrySubscriptionService tsSubService; + @Autowired + private ClusterRoutingService routingService; + + @Autowired + private ClusterRpcService clusterRpcService; + @Value("${state.defaultInactivityTimeoutInSec}") @Getter private long defaultInactivityTimeoutInSec; @@ -129,32 +148,18 @@ public class DefaultDeviceStateService implements DeviceStateService { private volatile boolean clusterUpdatePending = false; private ListeningScheduledExecutorService queueExecutor; - private final ConcurrentMap> partitionedDevices = new ConcurrentHashMap<>(); - private final ConcurrentMap deviceStates = new ConcurrentHashMap<>(); - private final ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); - private final ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); - - public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, - AttributesService attributesService, TimeseriesService tsService, - TbClusterService clusterService, PartitionService partitionService) { - this.tenantService = tenantService; - this.deviceService = deviceService; - this.attributesService = attributesService; - this.tsService = tsService; - this.clusterService = clusterService; - this.partitionService = partitionService; - } - - @Autowired - public void setTsSubService(TelemetrySubscriptionService tsSubService) { - this.tsSubService = tsSubService; - } + private ConcurrentMap> tenantDevices = new ConcurrentHashMap<>(); + private ConcurrentMap deviceStates = new ConcurrentHashMap<>(); + private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); + private ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); @PostConstruct public void init() { // Should be always single threaded due to absence of locks. queueExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-state"))); + queueExecutor.submit(this::initStateFromDB); queueExecutor.scheduleAtFixedRate(this::updateState, new Random().nextInt(defaultStateCheckIntervalInSec), defaultStateCheckIntervalInSec, TimeUnit.SECONDS); + //TODO: schedule persistence in v2.1; } @PreDestroy @@ -166,210 +171,133 @@ public class DefaultDeviceStateService implements DeviceStateService { @Override public void onDeviceAdded(Device device) { - sendDeviceEvent(device.getTenantId(), device.getId(), true, false, false); + queueExecutor.submit(() -> onDeviceAddedSync(device)); } @Override public void onDeviceUpdated(Device device) { - sendDeviceEvent(device.getTenantId(), device.getId(), false, true, false); + queueExecutor.submit(() -> onDeviceUpdatedSync(device)); } @Override - public void onDeviceDeleted(Device device) { - sendDeviceEvent(device.getTenantId(), device.getId(), false, false, true); + public void onDeviceConnect(DeviceId deviceId) { + queueExecutor.submit(() -> onDeviceConnectSync(deviceId)); } @Override - public void onDeviceConnect(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastConnectTime(ts); - pushRuleEngineMessage(stateData, CONNECT_EVENT); - save(deviceId, LAST_CONNECT_TIME, ts); - } + public void onDeviceActivity(DeviceId deviceId) { + deviceLastReportedActivity.put(deviceId, System.currentTimeMillis()); + queueExecutor.submit(() -> onDeviceActivitySync(deviceId)); } @Override - public void onDeviceActivity(DeviceId deviceId, long lastReportedActivity) { - deviceLastReportedActivity.put(deviceId, lastReportedActivity); - long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); - if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - DeviceState state = stateData.getState(); - stateData.getState().setLastActivityTime(lastReportedActivity); - stateData.getMetaData().putValue("scope", SERVER_SCOPE); - pushRuleEngineMessage(stateData, ACTIVITY_EVENT); - save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); - deviceLastSavedActivity.put(deviceId, lastReportedActivity); - if (!state.isActive()) { - state.setActive(true); - save(deviceId, ACTIVITY_STATE, state.isActive()); - } - } - } + public void onDeviceDisconnect(DeviceId deviceId) { + queueExecutor.submit(() -> onDeviceDisconnectSync(deviceId)); } @Override - public void onDeviceDisconnect(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastDisconnectTime(ts); - pushRuleEngineMessage(stateData, DISCONNECT_EVENT); - save(deviceId, LAST_DISCONNECT_TIME, ts); - } + public void onDeviceDeleted(Device device) { + queueExecutor.submit(() -> onDeviceDeleted(device.getTenantId(), device.getId())); } @Override public void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { - if (inactivityTimeout == 0L) { - return; - } - DeviceStateData stateData = deviceStates.get(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - DeviceState state = stateData.getState(); - state.setInactivityTimeout(inactivityTimeout); - boolean oldActive = state.isActive(); - state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); - if (!oldActive && state.isActive() || oldActive && !state.isActive()) { - save(deviceId, ACTIVITY_STATE, state.isActive()); - } + queueExecutor.submit(() -> onInactivityTimeoutUpdate(deviceId, inactivityTimeout)); + } + + @Override + public void onClusterUpdate() { + if (!clusterUpdatePending) { + clusterUpdatePending = true; + queueExecutor.submit(this::onClusterUpdateSync); } } @Override - public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbCallback callback) { + public void onRemoteMsg(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.DeviceStateServiceMsgProto proto; try { - TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); - if (proto.getDeleted()) { - onDeviceDeleted(tenantId, deviceId); - callback.onSuccess(); - } else { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - if (proto.getAdded()) { - Futures.addCallback(fetchDeviceState(device), new FutureCallback() { - @Override - public void onSuccess(@Nullable DeviceStateData state) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, device.getId()); - if (partitionedDevices.containsKey(tpi)) { - addDeviceUsingState(tpi, state); - callback.onSuccess(); - } else { - log.warn("[{}][{}] Device belongs to external partition. Probably rebalancing is in progress. Topic: {}" - , tenantId, deviceId, tpi.getFullTopicName()); - callback.onFailure(new RuntimeException("Device belongs to external partition " + tpi.getFullTopicName() + "!")); - } - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to register device to the state service", t); - callback.onFailure(t); - } - }, MoreExecutors.directExecutor()); - } else if (proto.getUpdated()) { - DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); - if (stateData != null) { - TbMsgMetaData md = new TbMsgMetaData(); - md.putValue("deviceName", device.getName()); - md.putValue("deviceType", device.getType()); - stateData.setMetaData(md); - } - } - } else { - //Device was probably deleted while message was in queue; - callback.onSuccess(); + proto = ClusterAPIProtos.DeviceStateServiceMsgProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); + if (proto.getDeleted()) { + queueExecutor.submit(() -> onDeviceDeleted(tenantId, deviceId)); + } else { + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + if (proto.getAdded()) { + onDeviceAdded(device); + } else if (proto.getUpdated()) { + onDeviceUpdated(device); } } - } catch (Exception e) { - log.trace("Failed to process queue msg: [{}]", proto, e); - callback.onFailure(e); } } - @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { - synchronized (this) { - if (!clusterUpdatePending) { - clusterUpdatePending = true; - queueExecutor.submit(() -> { - clusterUpdatePending = false; - initStateFromDB(partitionChangeEvent.getPartitions()); - }); + private void onClusterUpdateSync() { + clusterUpdatePending = false; + List tenants = tenantService.findTenants(new PageLink(Integer.MAX_VALUE)).getData(); + for (Tenant tenant : tenants) { + List> fetchFutures = new ArrayList<>(); + PageLink pageLink = new PageLink(initFetchPackSize); + while (pageLink != null) { + PageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); + pageLink = page.hasNext() ? pageLink.nextPageLink() : null; + for (Device device : page.getData()) { + if (!routingService.resolveById(device.getId()).isPresent()) { + if (!deviceStates.containsKey(device.getId())) { + fetchFutures.add(fetchDeviceState(device)); + } + } else { + Set tenantDeviceSet = tenantDevices.get(tenant.getId()); + if (tenantDeviceSet != null) { + tenantDeviceSet.remove(device.getId()); + } + deviceStates.remove(device.getId()); + deviceLastReportedActivity.remove(device.getId()); + deviceLastSavedActivity.remove(device.getId()); + } + } + try { + Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); + } catch (InterruptedException | ExecutionException e) { + log.warn("Failed to init device state service from DB", e); } } } } - private void initStateFromDB(Set partitions) { + private void initStateFromDB() { try { - Set addedPartitions = new HashSet<>(partitions); - addedPartitions.removeAll(partitionedDevices.keySet()); - - Set removedPartitions = new HashSet<>(partitionedDevices.keySet()); - removedPartitions.removeAll(partitions); - - // We no longer manage current partition of devices; - removedPartitions.forEach(partition -> { - Set devices = partitionedDevices.remove(partition); - devices.forEach(deviceId -> { - deviceStates.remove(deviceId); - deviceLastReportedActivity.remove(deviceId); - deviceLastSavedActivity.remove(deviceId); - }); - }); - - addedPartitions.forEach(tpi -> partitionedDevices.computeIfAbsent(tpi, key -> ConcurrentHashMap.newKeySet())); - - //TODO 3.0: replace this dummy search with new functionality to search by partitions using SQL capabilities. - // Adding only devices that are in new partitions List tenants = tenantService.findTenants(new PageLink(Integer.MAX_VALUE)).getData(); for (Tenant tenant : tenants) { + List> fetchFutures = new ArrayList<>(); PageLink pageLink = new PageLink(initFetchPackSize); while (pageLink != null) { - List> fetchFutures = new ArrayList<>(); PageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.hasNext() ? pageLink.nextPageLink() : null; for (Device device : page.getData()) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), device.getId()); - if (addedPartitions.contains(tpi)) { - ListenableFuture future = Futures.transform(fetchDeviceState(device), new Function() { - @Nullable - @Override - public Void apply(@Nullable DeviceStateData state) { - if (state != null) { - addDeviceUsingState(tpi, state); - } - return null; - } - }, MoreExecutors.directExecutor()); - fetchFutures.add(future); + if (!routingService.resolveById(device.getId()).isPresent()) { + fetchFutures.add(fetchDeviceState(device)); } } try { - Futures.successfulAsList(fetchFutures).get(); + Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); } catch (InterruptedException | ExecutionException e) { log.warn("Failed to init device state service from DB", e); } } } - log.info("Managing following partitions:"); - partitionedDevices.forEach((tpi, devices) -> { - log.info("[{}]: {} devices", tpi.getFullTopicName(), devices.size()); - }); } catch (Throwable t) { log.warn("Failed to init device states from DB", t); } } - private void addDeviceUsingState(TopicPartitionInfo tpi, DeviceStateData state) { - partitionedDevices.computeIfAbsent(tpi, id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); + private void addDeviceUsingState(DeviceStateData state) { + tenantDevices.computeIfAbsent(state.getTenantId(), id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); deviceStates.put(state.getDeviceId(), state); } @@ -397,24 +325,103 @@ public class DefaultDeviceStateService implements DeviceStateService { } } + private void onDeviceConnectSync(DeviceId deviceId) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastConnectTime(ts); + pushRuleEngineMessage(stateData, CONNECT_EVENT); + save(deviceId, LAST_CONNECT_TIME, ts); + } + } + + private void onDeviceDisconnectSync(DeviceId deviceId) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastDisconnectTime(ts); + pushRuleEngineMessage(stateData, DISCONNECT_EVENT); + save(deviceId, LAST_DISCONNECT_TIME, ts); + } + } + + private void onDeviceActivitySync(DeviceId deviceId) { + long lastReportedActivity = deviceLastReportedActivity.getOrDefault(deviceId, 0L); + long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); + if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + DeviceState state = stateData.getState(); + stateData.getState().setLastActivityTime(lastReportedActivity); + stateData.getMetaData().putValue("scope", SERVER_SCOPE); + pushRuleEngineMessage(stateData, ACTIVITY_EVENT); + save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); + deviceLastSavedActivity.put(deviceId, lastReportedActivity); + if (!state.isActive()) { + state.setActive(true); + save(deviceId, ACTIVITY_STATE, state.isActive()); + } + } + } + } + private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) { DeviceStateData deviceStateData = deviceStates.get(deviceId); if (deviceStateData == null) { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - try { - deviceStateData = fetchDeviceState(device).get(); - deviceStates.putIfAbsent(deviceId, deviceStateData); - } catch (InterruptedException | ExecutionException e) { - log.debug("[{}] Failed to fetch device state!", deviceId, e); + if (!routingService.resolveById(deviceId).isPresent()) { + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + try { + deviceStateData = fetchDeviceState(device).get(); + deviceStates.putIfAbsent(deviceId, deviceStateData); + } catch (InterruptedException | ExecutionException e) { + log.debug("[{}] Failed to fetch device state!", deviceId, e); + } } } } return deviceStateData; } - private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, boolean added, boolean updated, boolean deleted) { - TransportProtos.DeviceStateServiceMsgProto.Builder builder = TransportProtos.DeviceStateServiceMsgProto.newBuilder(); + private void onInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { + if (inactivityTimeout == 0L) { + return; + } + DeviceStateData stateData = deviceStates.get(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + DeviceState state = stateData.getState(); + state.setInactivityTimeout(inactivityTimeout); + boolean oldActive = state.isActive(); + state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); + if (!oldActive && state.isActive() || oldActive && !state.isActive()) { + save(deviceId, ACTIVITY_STATE, state.isActive()); + } + } + } + + private void onDeviceAddedSync(Device device) { + Optional address = routingService.resolveById(device.getId()); + if (!address.isPresent()) { + Futures.addCallback(fetchDeviceState(device), new FutureCallback() { + @Override + public void onSuccess(@Nullable DeviceStateData state) { + addDeviceUsingState(state); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to register device to the state service", t); + } + }, MoreExecutors.directExecutor()); + } else { + sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); + } + } + + private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, ServerAddress address, boolean added, boolean updated, boolean deleted) { + log.trace("[{}][{}] Device is monitored on other server: {}", tenantId, deviceId, address); + ClusterAPIProtos.DeviceStateServiceMsgProto.Builder builder = ClusterAPIProtos.DeviceStateServiceMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setDeviceIdMSB(deviceId.getId().getMostSignificantBits()); @@ -422,17 +429,40 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - TransportProtos.DeviceStateServiceMsgProto msg = builder.build(); - clusterService.pushMsgToCore(tenantId, deviceId, TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build(), null); + clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); + } + + private void onDeviceUpdatedSync(Device device) { + Optional address = routingService.resolveById(device.getId()); + if (!address.isPresent()) { + DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); + if (stateData != null) { + TbMsgMetaData md = new TbMsgMetaData(); + md.putValue("deviceName", device.getName()); + md.putValue("deviceType", device.getType()); + stateData.setMetaData(md); + } + } else { + sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); + } } private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) { - deviceStates.remove(deviceId); - deviceLastReportedActivity.remove(deviceId); - deviceLastSavedActivity.remove(deviceId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); - Set deviceIdSet = partitionedDevices.get(tpi); - deviceIdSet.remove(deviceId); + Optional address = routingService.resolveById(deviceId); + if (!address.isPresent()) { + deviceStates.remove(deviceId); + deviceLastReportedActivity.remove(deviceId); + deviceLastSavedActivity.remove(deviceId); + Set deviceIds = tenantDevices.get(tenantId); + if (deviceIds != null) { + deviceIds.remove(deviceId); + if (deviceIds.isEmpty()) { + tenantDevices.remove(tenantId); + } + } + } else { + sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); + } } private ListenableFuture fetchDeviceState(Device device) { @@ -493,9 +523,10 @@ public class DefaultDeviceStateService implements DeviceStateService { private void pushRuleEngineMessage(DeviceStateData stateData, String msgType) { DeviceState state = stateData.getState(); try { - TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON - , json.writeValueAsString(state)); - clusterService.pushMsgToRuleEngine(stateData.getTenantId(), stateData.getDeviceId(), tbMsg, null); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON + , json.writeValueAsString(state) + , null, null, 0L); + actorService.onMsg(new SendToClusterMsg(stateData.getDeviceId(), new ServiceToRuleEngineMsg(stateData.getTenantId(), tbMsg))); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } @@ -523,7 +554,7 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private static class AttributeSaveCallback implements FutureCallback { + private class AttributeSaveCallback implements FutureCallback { private final DeviceId deviceId; private final String key; private final Object value; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 1665a27b84..8dbe7579e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -15,17 +15,14 @@ */ package org.thingsboard.server.service.state; -import org.springframework.context.ApplicationListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.cluster.ServerAddress; /** * Created by ashvayka on 01.05.18. */ -public interface DeviceStateService extends ApplicationListener { +public interface DeviceStateService { void onDeviceAdded(Device device); @@ -35,12 +32,13 @@ public interface DeviceStateService extends ApplicationListener CALLBACK = new FutureCallback() { - @Override - public void onSuccess(@Nullable Void result) { - - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to persist statistics", t); - } - }; - - private final TbServiceInfoProvider serviceInfoProvider; - private final TelemetrySubscriptionService tsService; - private final Lock lock = new ReentrantLock(); - private final AssetService assetService; - private final ConcurrentMap tenantQueueAssets; - - public DefaultRuleEngineStatisticsService(TelemetrySubscriptionService tsService, TbServiceInfoProvider serviceInfoProvider, AssetService assetService) { - this.tsService = tsService; - this.serviceInfoProvider = serviceInfoProvider; - this.assetService = assetService; - this.tenantQueueAssets = new ConcurrentHashMap<>(); - } - - @Override - public void reportQueueStats(long ts, TbRuleEngineConsumerStats ruleEngineStats) { - String queueName = ruleEngineStats.getQueueName(); - ruleEngineStats.getTenantStats().forEach((id, stats) -> { - TenantId tenantId = new TenantId(id); - try { - AssetId serviceAssetId = getServiceAssetId(tenantId, queueName); - if (stats.getTotalMsgCounter().get() > 0) { - List tsList = stats.getCounters().entrySet().stream() - .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) - .collect(Collectors.toList()); - if (!tsList.isEmpty()) { - tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); - } - } - } catch (DataValidationException e) { - if (!e.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { - throw e; - } - } - }); - ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { - TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); - try { - tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); - } catch (DataValidationException e2) { - if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { - throw e2; - } - } - }); - ruleEngineStats.reset(); - } - - private AssetId getServiceAssetId(TenantId tenantId, String queueName) { - TenantQueueKey key = new TenantQueueKey(tenantId, queueName); - AssetId assetId = tenantQueueAssets.get(key); - if (assetId == null) { - lock.lock(); - try { - assetId = tenantQueueAssets.get(key); - if (assetId == null) { - Asset asset = assetService.findAssetByTenantIdAndName(tenantId, queueName + "_" + serviceInfoProvider.getServiceId()); - if (asset == null) { - asset = new Asset(); - asset.setTenantId(tenantId); - asset.setName(queueName + "_" + serviceInfoProvider.getServiceId()); - asset.setType(TB_SERVICE_QUEUE); - asset = assetService.saveAsset(asset); - } - assetId = asset.getId(); - tenantQueueAssets.put(key, assetId); - } - } finally { - lock.unlock(); - } - } - return assetId; - } - - @Data - private static class TenantQueueKey { - private final TenantId tenantId; - private final String queueName; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java deleted file mode 100644 index 88573844c9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.stats; - -import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; - -public interface RuleEngineStatisticsService { - - void reportQueueStats(long ts, TbRuleEngineConsumerStats stats); -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java deleted file mode 100644 index 67c81ac06f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ /dev/null @@ -1,380 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; -import org.thingsboard.common.util.DonAsynchron; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos.*; -import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateValueListProto; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.state.DefaultDeviceStateService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Function; -import java.util.function.Predicate; - -@Slf4j -@TbCoreComponent -@Service -public class DefaultSubscriptionManagerService implements SubscriptionManagerService { - - @Autowired - private AttributesService attrService; - - @Autowired - private TimeseriesService tsService; - - @Autowired - private PartitionService partitionService; - - @Autowired - private TbServiceInfoProvider serviceInfoProvider; - - @Autowired - private TbQueueProducerProvider producerProvider; - - @Autowired - private TbLocalSubscriptionService localSubscriptionService; - - @Autowired - private DeviceStateService deviceStateService; - - @Autowired - private TbClusterService clusterService; - - private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); - private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); - private final ConcurrentMap> partitionedSubscriptions = new ConcurrentHashMap<>(); - private final Set currentPartitions = ConcurrentHashMap.newKeySet(); - - private ExecutorService tsCallBackExecutor; - private String serviceId; - private TbQueueProducer> toCoreNotificationsProducer; - - @PostConstruct - public void initExecutor() { - tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); - serviceId = serviceInfoProvider.getServiceId(); - toCoreNotificationsProducer = producerProvider.getTbCoreNotificationsMsgProducer(); - } - - @PreDestroy - public void shutdownExecutor() { - if (tsCallBackExecutor != null) { - tsCallBackExecutor.shutdownNow(); - } - } - - @Override - public void addSubscription(TbSubscription subscription, TbCallback callback) { - log.trace("[{}][{}][{}] Registering remote subscription for entity [{}]", - subscription.getServiceId(), subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); - if (currentPartitions.contains(tpi)) { - partitionedSubscriptions.computeIfAbsent(tpi, k -> ConcurrentHashMap.newKeySet()).add(subscription); - callback.onSuccess(); - } else { - log.warn("[{}][{}] Entity belongs to external partition. Probably rebalancing is in progress. Topic: {}" - , subscription.getTenantId(), subscription.getEntityId(), tpi.getFullTopicName()); - callback.onFailure(new RuntimeException("Entity belongs to external partition " + tpi.getFullTopicName() + "!")); - } - boolean newSubscription = subscriptionsByEntityId - .computeIfAbsent(subscription.getEntityId(), k -> ConcurrentHashMap.newKeySet()).add(subscription); - subscriptionsByWsSessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()).put(subscription.getSubscriptionId(), subscription); - if (newSubscription) { - switch (subscription.getType()) { - case TIMESERIES: - handleNewTelemetrySubscription((TbTimeseriesSubscription) subscription); - break; - case ATTRIBUTES: - handleNewAttributeSubscription((TbAttributeSubscription) subscription); - break; - } - } - } - - @Override - public void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback) { - log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); - Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); - if (sessionSubscriptions != null) { - TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); - if (subscription != null) { - removeSubscriptionFromEntityMap(subscription); - removeSubscriptionFromPartitionMap(subscription); - if (sessionSubscriptions.isEmpty()) { - subscriptionsByWsSessionId.remove(sessionId); - } - } else { - log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); - } - } else { - log.debug("[{}] No session subscriptions found!", sessionId); - } - callback.onSuccess(); - } - - @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { - Set removedPartitions = new HashSet<>(currentPartitions); - removedPartitions.removeAll(partitionChangeEvent.getPartitions()); - - currentPartitions.clear(); - currentPartitions.addAll(partitionChangeEvent.getPartitions()); - - // We no longer manage current partition of devices; - removedPartitions.forEach(partition -> { - Set subs = partitionedSubscriptions.remove(partition); - if (subs != null) { - subs.forEach(this::removeSubscriptionFromEntityMap); - } - }); - } - } - - @Override - public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback) { - onLocalSubUpdate(entityId, - s -> { - if (TbSubscriptionType.TIMESERIES.equals(s.getType())) { - return (TbTimeseriesSubscription) s; - } else { - return null; - } - }, s -> true, s -> { - List subscriptionUpdate = null; - for (TsKvEntry kv : ts) { - if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(kv); - } - } - return subscriptionUpdate; - }); - callback.onSuccess(); - } - - @Override - public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback) { - onLocalSubUpdate(entityId, - s -> { - if (TbSubscriptionType.ATTRIBUTES.equals(s.getType())) { - return (TbAttributeSubscription) s; - } else { - return null; - } - }, - s -> (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope().name())), - s -> { - List subscriptionUpdate = null; - for (AttributeKvEntry kv : attributes) { - if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); - } - } - return subscriptionUpdate; - }); - if (entityId.getEntityType() == EntityType.DEVICE) { - if (TbAttributeSubscriptionScope.SERVER_SCOPE.name().equalsIgnoreCase(scope)) { - for (AttributeKvEntry attribute : attributes) { - if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { - deviceStateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); - } - } - } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope)) { - clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate(tenantId, - new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)) - , null); - } - } - callback.onSuccess(); - } - - private void onLocalSubUpdate(EntityId entityId, - Function castFunction, - Predicate filterFunction, - Function> processFunction) { - Set entitySubscriptions = subscriptionsByEntityId.get(entityId); - if (entitySubscriptions != null) { - entitySubscriptions.stream().map(castFunction).filter(Objects::nonNull).filter(filterFunction).forEach(s -> { - List subscriptionUpdate = processFunction.apply(s); - if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { - if (serviceId.equals(s.getServiceId())) { - SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); - localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbCallback.EMPTY); - } else { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId()); - toCoreNotificationsProducer.send(tpi, toProto(s, subscriptionUpdate), null); - } - } - }); - } else { - log.debug("[{}] No device subscriptions to process!", entityId); - } - } - - private boolean isInTimeRange(TbTimeseriesSubscription subscription, long kvTime) { - return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) - && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); - } - - private void removeSubscriptionFromEntityMap(TbSubscription sub) { - Set entitySubSet = subscriptionsByEntityId.get(sub.getEntityId()); - if (entitySubSet != null) { - entitySubSet.remove(sub); - if (entitySubSet.isEmpty()) { - subscriptionsByEntityId.remove(sub.getEntityId()); - } - } - } - - private void removeSubscriptionFromPartitionMap(TbSubscription sub) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, sub.getTenantId(), sub.getEntityId()); - Set subs = partitionedSubscriptions.get(tpi); - if (subs != null) { - subs.remove(sub); - } - } - - private void handleNewAttributeSubscription(TbAttributeSubscription subscription) { - log.trace("[{}][{}][{}] Processing remote attribute subscription for entity [{}]", - serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); - - final Map keyStates = subscription.getKeyStates(); - DonAsynchron.withCallback(attrService.find(subscription.getTenantId(), subscription.getEntityId(), DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { - List missedUpdates = new ArrayList<>(); - values.forEach(latestEntry -> { - if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { - missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); - } - }); - if (!missedUpdates.isEmpty()) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); - toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); - } - }, - e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); - } - - private void handleNewTelemetrySubscription(TbTimeseriesSubscription subscription) { - log.trace("[{}][{}][{}] Processing remote telemetry subscription for entity [{}]", - serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); - - long curTs = System.currentTimeMillis(); - List queries = new ArrayList<>(); - subscription.getKeyStates().forEach((key, value) -> { - if (curTs > value) { - long startTs = subscription.getStartTime() > 0 ? Math.max(subscription.getStartTime(), value + 1L) : (value + 1L); - long endTs = subscription.getEndTime() > 0 ? Math.min(subscription.getEndTime(), curTs) : curTs; - queries.add(new BaseReadTsKvQuery(key, startTs, endTs, 0, 1000, Aggregation.NONE)); - } - }); - if (!queries.isEmpty()) { - DonAsynchron.withCallback(tsService.findAll(subscription.getTenantId(), subscription.getEntityId(), queries), - missedUpdates -> { - if (missedUpdates != null && !missedUpdates.isEmpty()) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); - toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); - } - }, - e -> log.error("Failed to fetch missed updates.", e), - tsCallBackExecutor); - } - } - - private TbProtoQueueMsg toProto(TbSubscription subscription, List updates) { - TbSubscriptionUpdateProto.Builder builder = TbSubscriptionUpdateProto.newBuilder(); - - builder.setSessionId(subscription.getSessionId()); - builder.setSubscriptionId(subscription.getSubscriptionId()); - - Map> data = new TreeMap<>(); - for (TsKvEntry tsEntry : updates) { - List values = data.computeIfAbsent(tsEntry.getKey(), k -> new ArrayList<>()); - Object[] value = new Object[2]; - value[0] = tsEntry.getTs(); - value[1] = tsEntry.getValueAsString(); - values.add(value); - } - - data.forEach((key, value) -> { - TbSubscriptionUpdateValueListProto.Builder dataBuilder = TbSubscriptionUpdateValueListProto.newBuilder(); - dataBuilder.setKey(key); - value.forEach(v -> { - Object[] array = (Object[]) v; - dataBuilder.addTs((long) array[0]); - dataBuilder.addValue((String) array[1]); - }); - builder.addData(dataBuilder.build()); - }); - - ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setToLocalSubscriptionServiceMsg( - LocalSubscriptionServiceMsgProto.newBuilder().setSubUpdate(builder.build()).build()) - .build(); - return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java deleted file mode 100644 index d57067fa28..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ /dev/null @@ -1,228 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.entityview.EntityViewService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -@Slf4j -@TbCoreComponent -@Service -public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionService { - - private final Set currentPartitions = ConcurrentHashMap.newKeySet(); - private final Map> subscriptionsBySessionId = new ConcurrentHashMap<>(); - - @Autowired - private TelemetryWebSocketService wsService; - - @Autowired - private EntityViewService entityViewService; - - @Autowired - private PartitionService partitionService; - - @Autowired - private TbClusterService clusterService; - - @Autowired - @Lazy - private SubscriptionManagerService subscriptionManagerService; - - private ExecutorService wsCallBackExecutor; - - @PostConstruct - public void initExecutor() { - wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); - } - - @PreDestroy - public void shutdownExecutor() { - if (wsCallBackExecutor != null) { - wsCallBackExecutor.shutdownNow(); - } - } - - @Override - @EventListener(PartitionChangeEvent.class) - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { - currentPartitions.clear(); - currentPartitions.addAll(partitionChangeEvent.getPartitions()); - } - } - - @Override - @EventListener(ClusterTopologyChangeEvent.class) - public void onApplicationEvent(ClusterTopologyChangeEvent event) { - if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) { - /* - * If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again. - * Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart. - * Although this is resource consuming operation, it is cheaper than sending ping/pong commands periodically - * It is also cheaper then caching the subscriptions by entity id and then lookup of those caches every time we have new telemetry in SubscriptionManagerService. - * Even if we cache locally the list of active subscriptions by entity id, it is still time consuming operation to get them from cache - * Since number of subscriptions is usually much less then number of devices that are pushing data. - */ - subscriptionsBySessionId.values().forEach(map -> map.values() - .forEach(sub -> pushSubscriptionToManagerService(sub, false))); - } - } - - //TODO 3.1: replace null callbacks with callbacks from websocket service. - @Override - public void addSubscription(TbSubscription subscription) { - EntityId entityId = subscription.getEntityId(); - // Telemetry subscription on Entity Views are handled differently, because we need to allow only certain keys and time ranges; - if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TbSubscriptionType.TIMESERIES.equals(subscription.getType())) { - subscription = resolveEntityViewSubscription((TbTimeseriesSubscription) subscription); - } - pushSubscriptionToManagerService(subscription, true); - registerSubscription(subscription); - } - - private void pushSubscriptionToManagerService(TbSubscription subscription, boolean pushToLocalService) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); - if (currentPartitions.contains(tpi)) { - // Subscription is managed on the same server; - if (pushToLocalService) { - subscriptionManagerService.addSubscription(subscription, TbCallback.EMPTY); - } - } else { - // Push to the queue; - TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toNewSubscriptionProto(subscription); - clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); - } - } - - @Override - public void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback) { - TbSubscription subscription = subscriptionsBySessionId - .getOrDefault(sessionId, Collections.emptyMap()).get(update.getSubscriptionId()); - if (subscription != null) { - switch (subscription.getType()) { - case TIMESERIES: - TbTimeseriesSubscription tsSub = (TbTimeseriesSubscription) subscription; - update.getLatestValues().forEach((key, value) -> tsSub.getKeyStates().put(key, value)); - break; - case ATTRIBUTES: - TbAttributeSubscription attrSub = (TbAttributeSubscription) subscription; - update.getLatestValues().forEach((key, value) -> attrSub.getKeyStates().put(key, value)); - break; - } - wsService.sendWsMsg(sessionId, update); - } - callback.onSuccess(); - } - - @Override - public void cancelSubscription(String sessionId, int subscriptionId) { - log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); - Map sessionSubscriptions = subscriptionsBySessionId.get(sessionId); - if (sessionSubscriptions != null) { - TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); - if (subscription != null) { - if (sessionSubscriptions.isEmpty()) { - subscriptionsBySessionId.remove(sessionId); - } - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); - if (currentPartitions.contains(tpi)) { - // Subscription is managed on the same server; - subscriptionManagerService.cancelSubscription(sessionId, subscriptionId, TbCallback.EMPTY); - } else { - // Push to the queue; - TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toCloseSubscriptionProto(subscription); - clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); - } - } else { - log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); - } - } else { - log.debug("[{}] No session subscriptions found!", sessionId); - } - } - - @Override - public void cancelAllSessionSubscriptions(String sessionId) { - Map subscriptions = subscriptionsBySessionId.get(sessionId); - if (subscriptions != null) { - Set toRemove = new HashSet<>(subscriptions.keySet()); - toRemove.forEach(id -> cancelSubscription(sessionId, id)); - } - } - - private TbSubscription resolveEntityViewSubscription(TbTimeseriesSubscription subscription) { - EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(subscription.getEntityId().getId())); - - Map keyStates; - if (subscription.isAllKeys()) { - keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); - } else { - keyStates = subscription.getKeyStates().entrySet() - .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - return TbTimeseriesSubscription.builder() - .serviceId(subscription.getServiceId()) - .sessionId(subscription.getSessionId()) - .subscriptionId(subscription.getSubscriptionId()) - .tenantId(subscription.getTenantId()) - .entityId(entityView.getEntityId()) - .startTime(entityView.getStartTimeMs()) - .endTime(entityView.getEndTimeMs()) - .allKeys(false) - .keyStates(keyStates).build(); - } - - private void registerSubscription(TbSubscription subscription) { - Map sessionSubscriptions = subscriptionsBySessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()); - sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java deleted file mode 100644 index e20b707348..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import org.springframework.context.ApplicationListener; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.common.msg.queue.TbCallback; - -import java.util.List; - -public interface SubscriptionManagerService extends ApplicationListener { - - void addSubscription(TbSubscription subscription, TbCallback callback); - - void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback); - - void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback); - - void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java deleted file mode 100644 index 83a86efefc..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import lombok.Builder; -import lombok.Data; -import lombok.Getter; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.Map; - -public class TbAttributeSubscription extends TbSubscription { - - @Getter private final boolean allKeys; - @Getter private final Map keyStates; - @Getter private final TbAttributeSubscriptionScope scope; - - @Builder - public TbAttributeSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, - boolean allKeys, Map keyStates, TbAttributeSubscriptionScope scope) { - super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.ATTRIBUTES); - this.allKeys = allKeys; - this.keyStates = keyStates; - this.scope = scope; - } - - @Override - public boolean equals(Object o) { - return super.equals(o); - } - - @Override - public int hashCode() { - return super.hashCode(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java deleted file mode 100644 index 58fba24250..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; - -public interface TbLocalSubscriptionService { - - void addSubscription(TbSubscription subscription); - - void cancelSubscription(String sessionId, int subscriptionId); - - void cancelAllSessionSubscriptions(String sessionId); - - void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback); - - void onApplicationEvent(PartitionChangeEvent event); - - void onApplicationEvent(ClusterTopologyChangeEvent event); -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java deleted file mode 100644 index 22b37ff690..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.Objects; - -@Data -@AllArgsConstructor -public abstract class TbSubscription { - - private final String serviceId; - private final String sessionId; - private final int subscriptionId; - private final TenantId tenantId; - private final EntityId entityId; - private final TbSubscriptionType type; - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TbSubscription that = (TbSubscription) o; - return subscriptionId == that.subscriptionId && - sessionId.equals(that.sessionId) && - tenantId.equals(that.tenantId) && - entityId.equals(that.entityId) && - type == that.type; - } - - @Override - public int hashCode() { - return Objects.hash(sessionId, subscriptionId, tenantId, entityId, type); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java deleted file mode 100644 index d0b4d713f9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java +++ /dev/null @@ -1,247 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; -import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeSubscriptionProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionKetStateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesSubscriptionProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.UUID; - -public class TbSubscriptionUtils { - - public static ToCoreMsg toNewSubscriptionProto(TbSubscription subscription) { - SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); - TbSubscriptionProto subscriptionProto = TbSubscriptionProto.newBuilder() - .setServiceId(subscription.getServiceId()) - .setSessionId(subscription.getSessionId()) - .setSubscriptionId(subscription.getSubscriptionId()) - .setTenantIdMSB(subscription.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(subscription.getTenantId().getId().getLeastSignificantBits()) - .setEntityType(subscription.getEntityId().getEntityType().name()) - .setEntityIdMSB(subscription.getEntityId().getId().getMostSignificantBits()) - .setEntityIdLSB(subscription.getEntityId().getId().getLeastSignificantBits()).build(); - - switch (subscription.getType()) { - case TIMESERIES: - TbTimeseriesSubscription tSub = (TbTimeseriesSubscription) subscription; - TbTimeSeriesSubscriptionProto.Builder tSubProto = TbTimeSeriesSubscriptionProto.newBuilder() - .setSub(subscriptionProto) - .setAllKeys(tSub.isAllKeys()); - tSub.getKeyStates().forEach((key, value) -> tSubProto.addKeyStates( - TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); - tSubProto.setStartTime(tSub.getStartTime()); - tSubProto.setEndTime(tSub.getEndTime()); - msgBuilder.setTelemetrySub(tSubProto.build()); - break; - case ATTRIBUTES: - TbAttributeSubscription aSub = (TbAttributeSubscription) subscription; - TbAttributeSubscriptionProto.Builder aSubProto = TbAttributeSubscriptionProto.newBuilder() - .setSub(subscriptionProto) - .setAllKeys(aSub.isAllKeys()) - .setScope(aSub.getScope().name()); - aSub.getKeyStates().forEach((key, value) -> aSubProto.addKeyStates( - TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); - msgBuilder.setAttributeSub(aSubProto.build()); - break; - } - return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); - } - - public static ToCoreMsg toCloseSubscriptionProto(TbSubscription subscription) { - SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); - TbSubscriptionCloseProto closeProto = TbSubscriptionCloseProto.newBuilder() - .setSessionId(subscription.getSessionId()) - .setSubscriptionId(subscription.getSubscriptionId()).build(); - msgBuilder.setSubClose(closeProto); - return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); - } - - public static TbSubscription fromProto(TbAttributeSubscriptionProto attributeSub) { - TbSubscriptionProto subProto = attributeSub.getSub(); - TbAttributeSubscription.TbAttributeSubscriptionBuilder builder = TbAttributeSubscription.builder() - .serviceId(subProto.getServiceId()) - .sessionId(subProto.getSessionId()) - .subscriptionId(subProto.getSubscriptionId()) - .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) - .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); - - builder.scope(TbAttributeSubscriptionScope.valueOf(attributeSub.getScope())); - builder.allKeys(attributeSub.getAllKeys()); - Map keyStates = new HashMap<>(); - attributeSub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); - builder.keyStates(keyStates); - return builder.build(); - } - - public static TbSubscription fromProto(TbTimeSeriesSubscriptionProto telemetrySub) { - TbSubscriptionProto subProto = telemetrySub.getSub(); - TbTimeseriesSubscription.TbTimeseriesSubscriptionBuilder builder = TbTimeseriesSubscription.builder() - .serviceId(subProto.getServiceId()) - .sessionId(subProto.getSessionId()) - .subscriptionId(subProto.getSubscriptionId()) - .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) - .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); - - builder.allKeys(telemetrySub.getAllKeys()); - Map keyStates = new HashMap<>(); - telemetrySub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); - builder.startTime(telemetrySub.getStartTime()); - builder.endTime(telemetrySub.getEndTime()); - builder.keyStates(keyStates); - return builder.build(); - } - - public static SubscriptionUpdate fromProto(TbSubscriptionUpdateProto proto) { - if (proto.getErrorCode() > 0) { - return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); - } else { - Map> data = new TreeMap<>(); - proto.getDataList().forEach(v -> { - List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); - for (int i = 0; i < v.getTsCount(); i++) { - Object[] value = new Object[2]; - value[0] = v.getTs(i); - value[1] = v.getValue(i); - values.add(value); - } - }); - return new SubscriptionUpdate(proto.getSubscriptionId(), data); - } - } - - public static ToCoreMsg toTimeseriesUpdateProto(TenantId tenantId, EntityId entityId, List ts) { - TbTimeSeriesUpdateProto.Builder builder = TbTimeSeriesUpdateProto.newBuilder(); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); - SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); - msgBuilder.setTsUpdate(builder); - return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); - } - - public static ToCoreMsg toAttributesUpdateProto(TenantId tenantId, EntityId entityId, String scope, List attributes) { - TbAttributeUpdateProto.Builder builder = TbAttributeUpdateProto.newBuilder(); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setScope(scope); - attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); - - SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); - msgBuilder.setAttrUpdate(builder); - return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); - } - - private static TsKvProto.Builder toKeyValueProto(long ts, KvEntry attr) { - KeyValueProto.Builder dataBuilder = KeyValueProto.newBuilder(); - dataBuilder.setKey(attr.getKey()); - dataBuilder.setType(KeyValueType.forNumber(attr.getDataType().ordinal())); - switch (attr.getDataType()) { - case BOOLEAN: - attr.getBooleanValue().ifPresent(dataBuilder::setBoolV); - break; - case LONG: - attr.getLongValue().ifPresent(dataBuilder::setLongV); - break; - case DOUBLE: - attr.getDoubleValue().ifPresent(dataBuilder::setDoubleV); - break; - case JSON: - attr.getJsonValue().ifPresent(dataBuilder::setJsonV); - break; - case STRING: - attr.getStrValue().ifPresent(dataBuilder::setStringV); - break; - } - return TsKvProto.newBuilder().setTs(ts).setKv(dataBuilder); - } - - public static EntityId toEntityId(String entityType, long entityIdMSB, long entityIdLSB) { - return EntityIdFactory.getByTypeAndUuid(entityType, new UUID(entityIdMSB, entityIdLSB)); - } - - public static List toTsKvEntityList(List dataList) { - List result = new ArrayList<>(dataList.size()); - dataList.forEach(proto -> result.add(new BasicTsKvEntry(proto.getTs(), getKvEntry(proto.getKv())))); - return result; - } - - public static List toAttributeKvList(List dataList) { - List result = new ArrayList<>(dataList.size()); - dataList.forEach(proto -> result.add(new BaseAttributeKvEntry(getKvEntry(proto.getKv()), proto.getTs()))); - return result; - } - - private static KvEntry getKvEntry(KeyValueProto proto) { - KvEntry entry = null; - DataType type = DataType.values()[proto.getType().getNumber()]; - switch (type) { - case BOOLEAN: - entry = new BooleanDataEntry(proto.getKey(), proto.getBoolV()); - break; - case LONG: - entry = new LongDataEntry(proto.getKey(), proto.getLongV()); - break; - case DOUBLE: - entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleV()); - break; - case STRING: - entry = new StringDataEntry(proto.getKey(), proto.getStringV()); - break; - case JSON: - entry = new JsonDataEntry(proto.getKey(), proto.getJsonV()); - break; - } - return entry; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java deleted file mode 100644 index 0be63f7b65..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.subscription; - -import lombok.Builder; -import lombok.Getter; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.Map; - -public class TbTimeseriesSubscription extends TbSubscription { - - @Getter private final boolean allKeys; - @Getter private final Map keyStates; - @Getter private final long startTime; - @Getter private final long endTime; - - @Builder - public TbTimeseriesSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, - boolean allKeys, Map keyStates, long startTime, long endTime) { - super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.TIMESERIES); - this.allKeys = allKeys; - this.keyStates = keyStates; - this.startTime = startTime; - this.endTime = endTime; - } - - @Override - public boolean equals(Object o) { - return super.equals(o); - } - - @Override - public int hashCode() { - return super.hashCode(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 665a787e6d..d29e5de5b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -18,43 +18,73 @@ package org.thingsboard.server.service.telemetry; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.event.EventListener; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.subscription.SubscriptionManagerService; -import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.service.state.DefaultDeviceStateService; +import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.telemetry.sub.Subscription; +import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; +import org.thingsboard.server.service.telemetry.sub.SubscriptionState; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -63,36 +93,39 @@ import java.util.function.Consumer; @Slf4j public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptionService { - private final Set currentPartitions = ConcurrentHashMap.newKeySet(); + @Autowired + private TelemetryWebSocketService wsService; - private final AttributesService attrService; - private final TimeseriesService tsService; - private final TbClusterService clusterService; - private final PartitionService partitionService; - private Optional subscriptionManagerService; + @Autowired + private AttributesService attrService; - private ExecutorService tsCallBackExecutor; - private ExecutorService wsCallBackExecutor; + @Autowired + private TimeseriesService tsService; - public DefaultTelemetrySubscriptionService(AttributesService attrService, - TimeseriesService tsService, - TbClusterService clusterService, - PartitionService partitionService) { - this.attrService = attrService; - this.tsService = tsService; - this.clusterService = clusterService; - this.partitionService = partitionService; - } + @Autowired + private ClusterRoutingService routingService; - @Autowired(required = false) - public void setSubscriptionManagerService(Optional subscriptionManagerService) { - this.subscriptionManagerService = subscriptionManagerService; - } + @Autowired + private ClusterRpcService rpcService; + + @Autowired + private EntityViewService entityViewService; + + @Autowired + @Lazy + private DeviceStateService stateService; + + @Autowired + @Lazy + private ActorService actorService; + + private ExecutorService tsCallBackExecutor; + private ExecutorService wsCallBackExecutor; @PostConstruct public void initExecutor() { - tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); - wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ws-callback")); + tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); + wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); } @PreDestroy @@ -105,12 +138,64 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } } + private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); + private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); + @Override - @EventListener(PartitionChangeEvent.class) - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { - currentPartitions.clear(); - currentPartitions.addAll(partitionChangeEvent.getPartitions()); + public void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub) { + long startTime = 0L; + long endTime = 0L; + if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TelemetryFeature.TIMESERIES.equals(sub.getType())) { + EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(entityId.getId())); + entityId = entityView.getEntityId(); + startTime = entityView.getStartTimeMs(); + endTime = entityView.getEndTimeMs(); + sub = getUpdatedSubscriptionState(entityId, sub, entityView); + } + Optional server = routingService.resolveById(entityId); + Subscription subscription; + if (server.isPresent()) { + ServerAddress address = server.get(); + log.trace("[{}] Forwarding subscription [{}] for [{}] entity [{}] to [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId, address); + subscription = new Subscription(sub, true, address, startTime, endTime); + tellNewSubscription(address, sessionId, subscription); + } else { + log.trace("[{}] Registering local subscription [{}] for [{}] entity [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId); + subscription = new Subscription(sub, true, null, startTime, endTime); + } + registerSubscription(sessionId, entityId, subscription); + } + + private SubscriptionState getUpdatedSubscriptionState(EntityId entityId, SubscriptionState sub, EntityView entityView) { + Map keyStates; + if (sub.isAllKeys()) { + keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); + } else { + keyStates = sub.getKeyStates().entrySet() + .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + return new SubscriptionState(sub.getWsSessionId(), sub.getSubscriptionId(), sub.getTenantId(), entityId, sub.getType(), false, keyStates, sub.getScope()); + } + + @Override + public void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId) { + cleanupLocalWsSessionSubscriptions(sessionId); + } + + @Override + public void removeSubscription(String sessionId, int subscriptionId) { + log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); + Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); + if (sessionSubscriptions != null) { + Subscription subscription = sessionSubscriptions.remove(subscriptionId); + if (subscription != null) { + processSubscriptionRemoval(sessionId, sessionSubscriptions, subscription); + } else { + log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); + } + } else { + log.debug("[{}] No session subscriptions found!", sessionId); } } @@ -123,14 +208,14 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); + addWsCallback(saveFuture, success -> onTimeseriesUpdate(entityId, ts)); } @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, FutureCallback callback) { ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes)); + addWsCallback(saveFuture, success -> onAttributesUpdate(entityId, scope, attributes)); } @Override @@ -157,31 +242,352 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio , System.currentTimeMillis())), callback); } - private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); - if (currentPartitions.contains(tpi)) { - if (subscriptionManagerService.isPresent()) { - subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbCallback.EMPTY); + @Override + public void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes) { + DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate(tenantId, + deviceId, DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)); + actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + } + + @Override + public void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.SubscriptionProto proto; + try { + proto = ClusterAPIProtos.SubscriptionProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + Map statesMap = proto.getKeyStatesList().stream().collect( + Collectors.toMap(ClusterAPIProtos.SubscriptionKetStateProto::getKey, ClusterAPIProtos.SubscriptionKetStateProto::getTs)); + Subscription subscription = new Subscription( + new SubscriptionState(proto.getSessionId(), proto.getSubscriptionId(), + new TenantId(UUID.fromString(proto.getTenantId())), + EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), + TelemetryFeature.valueOf(proto.getType()), proto.getAllKeys(), statesMap, proto.getScope()), + false, new ServerAddress(serverAddress.getHost(), serverAddress.getPort(), serverAddress.getServerType())); + + addRemoteWsSubscription(serverAddress, proto.getSessionId(), subscription); + } + + @Override + public void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.SubscriptionUpdateProto proto; + try { + proto = ClusterAPIProtos.SubscriptionUpdateProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + SubscriptionUpdate update = convert(proto); + String sessionId = proto.getSessionId(); + log.trace("[{}] Processing remote subscription onUpdate [{}]", sessionId, update); + Optional subOpt = getSubscription(sessionId, update.getSubscriptionId()); + if (subOpt.isPresent()) { + updateSubscriptionState(sessionId, subOpt.get(), update); + wsService.sendWsMsg(sessionId, update); + } + } + + @Override + public void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.SubscriptionCloseProto proto; + try { + proto = ClusterAPIProtos.SubscriptionCloseProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + removeSubscription(proto.getSessionId(), proto.getSubscriptionId()); + } + + @Override + public void onRemoteSessionClose(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.SessionCloseProto proto; + try { + proto = ClusterAPIProtos.SessionCloseProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + cleanupRemoteWsSessionSubscriptions(proto.getSessionId()); + } + + @Override + public void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.AttributeUpdateProto proto; + try { + proto = ClusterAPIProtos.AttributeUpdateProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + onAttributesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), proto.getScope(), + proto.getDataList().stream().map(this::toAttribute).collect(Collectors.toList())); + } + + @Override + public void onRemoteTsUpdate(ServerAddress serverAddress, byte[] data) { + ClusterAPIProtos.TimeseriesUpdateProto proto; + try { + proto = ClusterAPIProtos.TimeseriesUpdateProto.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + onTimeseriesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), + proto.getDataList().stream().map(this::toTimeseries).collect(Collectors.toList())); + } + + @Override + public void onClusterUpdate() { + log.trace("Processing cluster onUpdate msg!"); + Iterator>> deviceIterator = subscriptionsByEntityId.entrySet().iterator(); + while (deviceIterator.hasNext()) { + Map.Entry> e = deviceIterator.next(); + Set subscriptions = e.getValue(); + Optional newAddressOptional = routingService.resolveById(e.getKey()); + if (newAddressOptional.isPresent()) { + newAddressOptional.ifPresent(serverAddress -> checkSubscriptionsNewAddress(serverAddress, subscriptions)); } else { - log.warn("Possible misconfiguration because subscriptionManagerService is null!"); + checkSubscriptionsPrevAddress(subscriptions); + } + if (subscriptions.size() == 0) { + log.trace("[{}] No more subscriptions for this device on current server.", e.getKey()); + deviceIterator.remove(); + } + } + } + + private void checkSubscriptionsNewAddress(ServerAddress newAddress, Set subscriptions) { + Iterator subscriptionIterator = subscriptions.iterator(); + while (subscriptionIterator.hasNext()) { + Subscription s = subscriptionIterator.next(); + if (s.isLocal()) { + if (!newAddress.equals(s.getServer())) { + log.trace("[{}] Local subscription is now handled on new server [{}]", s.getWsSessionId(), newAddress); + s.setServer(newAddress); + tellNewSubscription(newAddress, s.getWsSessionId(), s); + } + } else { + log.trace("[{}] Remote subscription is now handled on new server address: [{}]", s.getWsSessionId(), newAddress); + subscriptionIterator.remove(); + //TODO: onUpdate state of subscription by WsSessionId and other maps. + } + } + } + + private void checkSubscriptionsPrevAddress(Set subscriptions) { + for (Subscription s : subscriptions) { + if (s.isLocal() && s.getServer() != null) { + log.trace("[{}] Local subscription is no longer handled on remote server address [{}]", s.getWsSessionId(), s.getServer()); + s.setServer(null); + } else { + log.trace("[{}] Remote subscription is on up to date server address.", s.getWsSessionId()); + } + } + } + + private void addRemoteWsSubscription(ServerAddress address, String sessionId, Subscription subscription) { + EntityId entityId = subscription.getEntityId(); + log.trace("[{}] Registering remote subscription [{}] for entity [{}] to [{}]", sessionId, subscription.getSubscriptionId(), entityId, address); + registerSubscription(sessionId, entityId, subscription); + if (subscription.getType() == TelemetryFeature.ATTRIBUTES) { + final Map keyStates = subscription.getKeyStates(); + DonAsynchron.withCallback(attrService.find(subscription.getSub().getTenantId(), entityId, DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { + List missedUpdates = new ArrayList<>(); + values.forEach(latestEntry -> { + if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { + missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); + } + }); + if (!missedUpdates.isEmpty()) { + tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); + } + }, + e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); + } else if (subscription.getType() == TelemetryFeature.TIMESERIES) { + long curTs = System.currentTimeMillis(); + List queries = new ArrayList<>(); + subscription.getKeyStates().entrySet().forEach(e -> { + if (curTs > e.getValue()) { + queries.add(new BaseReadTsKvQuery(e.getKey(), e.getValue() + 1L, curTs, 0, 1000, Aggregation.NONE)); + } else { + log.debug("[{}] Invalid subscription [{}], entityId [{}] curTs [{}]", sessionId, subscription, entityId, curTs); + } + }); + if (!queries.isEmpty()) { + DonAsynchron.withCallback(tsService.findAll(subscription.getSub().getTenantId(), entityId, queries), + missedUpdates -> { + if (missedUpdates != null && !missedUpdates.isEmpty()) { + tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); + } + }, + e -> log.error("Failed to fetch missed updates.", e), + tsCallBackExecutor); + } + } + } + + private void onAttributesUpdate(EntityId entityId, String scope, List attributes) { + Optional serverAddress = routingService.resolveById(entityId); + if (!serverAddress.isPresent()) { + onLocalAttributesUpdate(entityId, scope, attributes); + if (entityId.getEntityType() == EntityType.DEVICE && DataConstants.SERVER_SCOPE.equalsIgnoreCase(scope)) { + for (AttributeKvEntry attribute : attributes) { + if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { + stateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); + } + } } } else { - TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); - clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); + tellRemoteAttributesUpdate(serverAddress.get(), entityId, scope, attributes); + } + } + + private void onTimeseriesUpdate(EntityId entityId, List ts) { + Optional serverAddress = routingService.resolveById(entityId); + if (!serverAddress.isPresent()) { + onLocalTimeseriesUpdate(entityId, ts); + } else { + tellRemoteTimeseriesUpdate(serverAddress.get(), entityId, ts); + } + } + + private void onLocalAttributesUpdate(EntityId entityId, String scope, List attributes) { + onLocalSubUpdate(entityId, s -> TelemetryFeature.ATTRIBUTES == s.getType() && (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope())), s -> { + List subscriptionUpdate = null; + for (AttributeKvEntry kv : attributes) { + if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); + } + } + return subscriptionUpdate; + }); + } + + private void onLocalTimeseriesUpdate(EntityId entityId, List ts) { + onLocalSubUpdate(entityId, s -> TelemetryFeature.TIMESERIES == s.getType(), s -> { + List subscriptionUpdate = null; + for (TsKvEntry kv : ts) { + if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(kv); + } + } + return subscriptionUpdate; + }); + } + + private boolean isInTimeRange(Subscription subscription, long kvTime) { + return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) + && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); + } + + private void onLocalSubUpdate(EntityId entityId, Predicate filter, Function> f) { + Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); + if (deviceSubscriptions != null) { + deviceSubscriptions.stream().filter(filter).forEach(s -> { + String sessionId = s.getWsSessionId(); + List subscriptionUpdate = f.apply(s); + if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { + SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); + if (s.isLocal()) { + updateSubscriptionState(sessionId, s, update); + wsService.sendWsMsg(sessionId, update); + } else { + tellRemoteSubUpdate(s.getServer(), sessionId, update); + } + } + }); + } else { + log.debug("[{}] No device subscriptions to process!", entityId); + } + } + + private void updateSubscriptionState(String sessionId, Subscription subState, SubscriptionUpdate update) { + log.trace("[{}] updating subscription state {} using onUpdate {}", sessionId, subState, update); + update.getLatestValues().entrySet().forEach(e -> subState.setKeyState(e.getKey(), e.getValue())); + } + + private void registerSubscription(String sessionId, EntityId entityId, Subscription subscription) { + Set deviceSubscriptions = subscriptionsByEntityId.computeIfAbsent(entityId, k -> ConcurrentHashMap.newKeySet()); + deviceSubscriptions.add(subscription); + Map sessionSubscriptions = subscriptionsByWsSessionId.computeIfAbsent(sessionId, k -> new ConcurrentHashMap<>()); + sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); + } + + private void cleanupLocalWsSessionSubscriptions(String sessionId) { + cleanupWsSessionSubscriptions(sessionId, true); + } + + private void cleanupRemoteWsSessionSubscriptions(String sessionId) { + cleanupWsSessionSubscriptions(sessionId, false); + } + + private void cleanupWsSessionSubscriptions(String sessionId, boolean localSession) { + log.debug("[{}] Removing all subscriptions for particular session.", sessionId); + Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); + if (sessionSubscriptions != null) { + int sessionSubscriptionSize = sessionSubscriptions.size(); + + for (Subscription subscription : sessionSubscriptions.values()) { + EntityId entityId = subscription.getEntityId(); + Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); + deviceSubscriptions.remove(subscription); + if (deviceSubscriptions.isEmpty()) { + subscriptionsByEntityId.remove(entityId); + } + } + subscriptionsByWsSessionId.remove(sessionId); + log.debug("[{}] Removed {} subscriptions for particular session.", sessionId, sessionSubscriptionSize); + + if (localSession) { + notifyWsSubscriptionClosed(sessionId, sessionSubscriptions); + } + } else { + log.debug("[{}] No subscriptions found!", sessionId); + } + } + + private void notifyWsSubscriptionClosed(String sessionId, Map sessionSubscriptions) { + Set affectedServers = new HashSet<>(); + for (Subscription subscription : sessionSubscriptions.values()) { + if (subscription.getServer() != null) { + affectedServers.add(subscription.getServer()); + } + } + for (ServerAddress address : affectedServers) { + log.debug("[{}] Going to onSubscriptionUpdate [{}] server about session close event", sessionId, address); + tellRemoteSessionClose(address, sessionId); } } - private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); - if (currentPartitions.contains(tpi)) { - if (subscriptionManagerService.isPresent()) { - subscriptionManagerService.get().onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY); + private void processSubscriptionRemoval(String sessionId, Map sessionSubscriptions, Subscription subscription) { + EntityId entityId = subscription.getEntityId(); + if (subscription.isLocal() && subscription.getServer() != null) { + tellRemoteSubClose(subscription.getServer(), sessionId, subscription.getSubscriptionId()); + } + if (sessionSubscriptions.isEmpty()) { + log.debug("[{}] Removed last subscription for particular session.", sessionId); + subscriptionsByWsSessionId.remove(sessionId); + } else { + log.debug("[{}] Removed session subscription.", sessionId); + } + Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); + if (deviceSubscriptions != null) { + boolean result = deviceSubscriptions.remove(subscription); + if (result) { + if (deviceSubscriptions.size() == 0) { + log.debug("[{}] Removed last subscription for particular device.", sessionId); + subscriptionsByEntityId.remove(entityId); + } else { + log.debug("[{}] Removed device subscription.", sessionId); + } } else { - log.warn("Possible misconfiguration because subscriptionManagerService is null!"); + log.debug("[{}] Subscription not found!", sessionId); } } else { - TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); - clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); + log.debug("[{}] No device subscriptions found!", sessionId); } } @@ -211,4 +617,161 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } }, wsCallBackExecutor); } + + private void tellNewSubscription(ServerAddress address, String sessionId, Subscription sub) { + ClusterAPIProtos.SubscriptionProto.Builder builder = ClusterAPIProtos.SubscriptionProto.newBuilder(); + builder.setSessionId(sessionId); + builder.setSubscriptionId(sub.getSubscriptionId()); + builder.setTenantId(sub.getSub().getTenantId().getId().toString()); + builder.setEntityType(sub.getEntityId().getEntityType().name()); + builder.setEntityId(sub.getEntityId().getId().toString()); + builder.setType(sub.getType().name()); + builder.setAllKeys(sub.isAllKeys()); + if (sub.getScope() != null) { + builder.setScope(sub.getScope()); + } + sub.getKeyStates().entrySet().forEach(e -> builder.addKeyStates( + ClusterAPIProtos.SubscriptionKetStateProto.newBuilder().setKey(e.getKey()).setTs(e.getValue()).build())); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); + } + + private void tellRemoteSubUpdate(ServerAddress address, String sessionId, SubscriptionUpdate update) { + ClusterAPIProtos.SubscriptionUpdateProto.Builder builder = ClusterAPIProtos.SubscriptionUpdateProto.newBuilder(); + builder.setSessionId(sessionId); + builder.setSubscriptionId(update.getSubscriptionId()); + builder.setErrorCode(update.getErrorCode()); + if (update.getErrorMsg() != null) { + builder.setErrorMsg(update.getErrorMsg()); + } + update.getData().entrySet().forEach( + e -> { + ClusterAPIProtos.SubscriptionUpdateValueListProto.Builder dataBuilder = ClusterAPIProtos.SubscriptionUpdateValueListProto.newBuilder(); + + dataBuilder.setKey(e.getKey()); + e.getValue().forEach(v -> { + Object[] array = (Object[]) v; + dataBuilder.addTs((long) array[0]); + dataBuilder.addValue((String) array[1]); + }); + + builder.addData(dataBuilder.build()); + } + ); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); + } + + private void tellRemoteAttributesUpdate(ServerAddress address, EntityId entityId, String scope, List attributes) { + ClusterAPIProtos.AttributeUpdateProto.Builder builder = ClusterAPIProtos.AttributeUpdateProto.newBuilder(); + builder.setEntityId(entityId.getId().toString()); + builder.setEntityType(entityId.getEntityType().name()); + builder.setScope(scope); + attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); + } + + private void tellRemoteTimeseriesUpdate(ServerAddress address, EntityId entityId, List ts) { + ClusterAPIProtos.TimeseriesUpdateProto.Builder builder = ClusterAPIProtos.TimeseriesUpdateProto.newBuilder(); + builder.setEntityId(entityId.getId().toString()); + builder.setEntityType(entityId.getEntityType().name()); + ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); + } + + private void tellRemoteSessionClose(ServerAddress address, String sessionId) { + ClusterAPIProtos.SessionCloseProto proto = ClusterAPIProtos.SessionCloseProto.newBuilder().setSessionId(sessionId).build(); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); + } + + private void tellRemoteSubClose(ServerAddress address, String sessionId, int subscriptionId) { + ClusterAPIProtos.SubscriptionCloseProto proto = ClusterAPIProtos.SubscriptionCloseProto.newBuilder().setSessionId(sessionId).setSubscriptionId(subscriptionId).build(); + rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); + } + + private ClusterAPIProtos.KeyValueProto.Builder toKeyValueProto(long ts, KvEntry attr) { + ClusterAPIProtos.KeyValueProto.Builder dataBuilder = ClusterAPIProtos.KeyValueProto.newBuilder(); + dataBuilder.setKey(attr.getKey()); + dataBuilder.setTs(ts); + dataBuilder.setValueType(attr.getDataType().ordinal()); + switch (attr.getDataType()) { + case BOOLEAN: + Optional booleanValue = attr.getBooleanValue(); + booleanValue.ifPresent(dataBuilder::setBoolValue); + break; + case LONG: + Optional longValue = attr.getLongValue(); + longValue.ifPresent(dataBuilder::setLongValue); + break; + case DOUBLE: + Optional doubleValue = attr.getDoubleValue(); + doubleValue.ifPresent(dataBuilder::setDoubleValue); + break; + case JSON: + Optional jsonValue = attr.getJsonValue(); + jsonValue.ifPresent(dataBuilder::setJsonValue); + break; + case STRING: + Optional stringValue = attr.getStrValue(); + stringValue.ifPresent(dataBuilder::setStrValue); + break; + } + return dataBuilder; + } + + private AttributeKvEntry toAttribute(ClusterAPIProtos.KeyValueProto proto) { + return new BaseAttributeKvEntry(getKvEntry(proto), proto.getTs()); + } + + private TsKvEntry toTimeseries(ClusterAPIProtos.KeyValueProto proto) { + return new BasicTsKvEntry(proto.getTs(), getKvEntry(proto)); + } + + private KvEntry getKvEntry(ClusterAPIProtos.KeyValueProto proto) { + KvEntry entry = null; + DataType type = DataType.values()[proto.getValueType()]; + switch (type) { + case BOOLEAN: + entry = new BooleanDataEntry(proto.getKey(), proto.getBoolValue()); + break; + case LONG: + entry = new LongDataEntry(proto.getKey(), proto.getLongValue()); + break; + case DOUBLE: + entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleValue()); + break; + case STRING: + entry = new StringDataEntry(proto.getKey(), proto.getStrValue()); + break; + case JSON: + entry = new JsonDataEntry(proto.getKey(), proto.getJsonValue()); + break; + } + return entry; + } + + private SubscriptionUpdate convert(ClusterAPIProtos.SubscriptionUpdateProto proto) { + if (proto.getErrorCode() > 0) { + return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); + } else { + Map> data = new TreeMap<>(); + proto.getDataList().forEach(v -> { + List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); + for (int i = 0; i < v.getTsCount(); i++) { + Object[] value = new Object[2]; + value[0] = v.getTs(i); + value[1] = v.getValue(i); + values.add(value); + } + }); + return new SubscriptionUpdate(proto.getSubscriptionId(), data); + } + } + + private Optional getSubscription(String sessionId, int subscriptionId) { + Subscription state = null; + Map subMap = subscriptionsByWsSessionId.get(sessionId); + if (subMap != null) { + state = subMap.get(subscriptionId); + } + return Optional.ofNullable(state); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 6afcaf97bb..741efe6f8a 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -43,18 +43,12 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.TenantRateLimitException; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.ValidationCallback; import org.thingsboard.server.service.security.ValidationResult; import org.thingsboard.server.service.security.ValidationResultCode; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; -import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; -import org.thingsboard.server.service.subscription.TbAttributeSubscription; -import org.thingsboard.server.service.subscription.TbTimeseriesSubscription; import org.thingsboard.server.service.telemetry.cmd.AttributesSubscriptionCmd; import org.thingsboard.server.service.telemetry.cmd.GetHistoryCmd; import org.thingsboard.server.service.telemetry.cmd.SubscriptionCmd; @@ -63,6 +57,7 @@ import org.thingsboard.server.service.telemetry.cmd.TelemetryPluginCmdsWrapper; import org.thingsboard.server.service.telemetry.cmd.TimeseriesSubscriptionCmd; import org.thingsboard.server.service.telemetry.exception.UnauthorizedException; import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; +import org.thingsboard.server.service.telemetry.sub.SubscriptionState; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; import javax.annotation.Nullable; @@ -88,7 +83,6 @@ import java.util.stream.Collectors; * Created by ashvayka on 27.03.18. */ @Service -@TbCoreComponent @Slf4j public class DefaultTelemetryWebSocketService implements TelemetryWebSocketService { @@ -104,7 +98,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private final ConcurrentMap wsSessionsMap = new ConcurrentHashMap<>(); @Autowired - private TbLocalSubscriptionService subService; + private TelemetrySubscriptionService subscriptionManager; @Autowired private TelemetryWebSocketMsgEndpoint msgEndpoint; @@ -118,9 +112,6 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Autowired private TimeseriesService tsService; - @Autowired - private TbServiceInfoProvider serviceInfoProvider; - @Value("${server.ws.limits.max_subscriptions_per_tenant:0}") private int maxSubscriptionsPerTenant; @Value("${server.ws.limits.max_subscriptions_per_customer:0}") @@ -136,11 +127,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private ConcurrentMap> publicUserSubscriptionsMap = new ConcurrentHashMap<>(); private ExecutorService executor; - private String serviceId; @PostConstruct public void initExecutor() { - serviceId = serviceInfoProvider.getServiceId(); executor = Executors.newWorkStealingPool(50); } @@ -164,7 +153,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi break; case CLOSED: wsSessionsMap.remove(sessionId); - subService.cancelAllSessionSubscriptions(sessionId); + subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); processSessionClose(sessionRef); break; } @@ -345,18 +334,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi keys.forEach(key -> subState.put(key, 0L)); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); - - TbAttributeSubscription sub = TbAttributeSubscription.builder() - .serviceId(serviceId) - .sessionId(sessionId) - .subscriptionId(cmd.getCmdId()) - .tenantId(sessionRef.getSecurityCtx().getTenantId()) - .entityId(entityId) - .allKeys(false) - .keyStates(subState) - .scope(scope).build(); - subService.addSubscription(sub); + SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, false, subState, cmd.getScope()); + subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); } @Override @@ -442,18 +421,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi Map subState = new HashMap<>(attributesData.size()); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); - - TbAttributeSubscription sub = TbAttributeSubscription.builder() - .serviceId(serviceId) - .sessionId(sessionId) - .subscriptionId(cmd.getCmdId()) - .tenantId(sessionRef.getSecurityCtx().getTenantId()) - .entityId(entityId) - .allKeys(true) - .keyStates(subState) - .scope(scope).build(); - subService.addSubscription(sub); + SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, true, subState, cmd.getScope()); + subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); } @Override @@ -525,16 +494,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); Map subState = new HashMap<>(data.size()); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - - TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() - .serviceId(serviceId) - .sessionId(sessionId) - .subscriptionId(cmd.getCmdId()) - .tenantId(sessionRef.getSecurityCtx().getTenantId()) - .entityId(entityId) - .allKeys(true) - .keyStates(subState).build(); - subService.addSubscription(sub); + SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, true, subState, cmd.getScope()); + subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); } @Override @@ -559,19 +520,12 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Override public void onSuccess(List data) { sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); + Map subState = new HashMap<>(keys.size()); keys.forEach(key -> subState.put(key, startTs)); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - - TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() - .serviceId(serviceId) - .sessionId(sessionId) - .subscriptionId(cmd.getCmdId()) - .tenantId(sessionRef.getSecurityCtx().getTenantId()) - .entityId(entityId) - .allKeys(false) - .keyStates(subState).build(); - subService.addSubscription(sub); + SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, false, subState, cmd.getScope()); + subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); } @Override @@ -590,9 +544,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private void unsubscribe(TelemetryWebSocketSessionRef sessionRef, SubscriptionCmd cmd, String sessionId) { if (cmd.getEntityId() == null || cmd.getEntityId().isEmpty()) { - subService.cancelAllSessionSubscriptions(sessionId); + subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); } else { - subService.cancelSubscription(sessionId, cmd.getCmdId()); + subscriptionManager.removeSubscription(sessionId, cmd.getCmdId()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index d652a3f1c5..d77a0e50dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -15,13 +15,34 @@ */ package org.thingsboard.server.service.telemetry; -import org.springframework.context.ApplicationListener; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.service.telemetry.sub.SubscriptionState; /** * Created by ashvayka on 27.03.18. */ -public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener { +public interface TelemetrySubscriptionService extends RuleEngineTelemetryService { + void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub); + + void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId); + + void removeSubscription(String sessionId, int cmdId); + + void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data); + + void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] bytes); + + void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] bytes); + + void onRemoteSessionClose(ServerAddress serverAddress, byte[] bytes); + + void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] bytes); + + void onRemoteTsUpdate(ServerAddress serverAddress, byte[] bytes); + + void onClusterUpdate(); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java new file mode 100644 index 0000000000..7f7db9430d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry.sub; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.service.telemetry.TelemetryFeature; + +import java.util.Map; + +@Data +@AllArgsConstructor +public class Subscription { + + private final SubscriptionState sub; + private final boolean local; + private ServerAddress server; + private long startTime; + private long endTime; + + public Subscription(SubscriptionState sub, boolean local, ServerAddress server) { + this(sub, local, server, 0L, 0L); + } + + public String getWsSessionId() { + return getSub().getWsSessionId(); + } + + public int getSubscriptionId() { + return getSub().getSubscriptionId(); + } + + public EntityId getEntityId() { + return getSub().getEntityId(); + } + + public TelemetryFeature getType() { + return getSub().getType(); + } + + public String getScope() { + return getSub().getScope(); + } + + public boolean isAllKeys() { + return getSub().isAllKeys(); + } + + public Map getKeyStates() { + return getSub().getKeyStates(); + } + + public void setKeyState(String key, long ts) { + getSub().getKeyStates().put(key, ts); + } + + @Override + public String toString() { + return "Subscription{" + + "sub=" + sub + + ", local=" + local + + ", server=" + server + + '}'; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java index 992dc26af9..68aaca34c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java @@ -75,7 +75,7 @@ public class SubscriptionUpdate { if (data == null) { return Collections.emptyMap(); } else { - return data.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> { + return data.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> { List data = e.getValue(); Object[] latest = (Object[]) data.get(data.size() - 1); return (long) latest[0]; diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java new file mode 100644 index 0000000000..1aee532c93 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java @@ -0,0 +1,242 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transaction; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RuleChainTransactionService; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.service.executors.DbCallbackExecutorService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; + +@Service +@Slf4j +public class BaseRuleChainTransactionService implements RuleChainTransactionService { + + @Autowired + private ClusterRoutingService routingService; + + @Autowired + private ClusterRpcService clusterRpcService; + + @Autowired + private DbCallbackExecutorService callbackExecutor; + + @Value("${actors.rule.transaction.queue_size}") + private int finalQueueSize; + @Value("${actors.rule.transaction.duration}") + private long duration; + + private final Lock transactionLock = new ReentrantLock(); + private final ConcurrentMap> transactionMap = new ConcurrentHashMap<>(); + private final Queue timeoutQueue = new ConcurrentLinkedQueue<>(); + + private ExecutorService timeoutExecutor; + + @PostConstruct + public void init() { + timeoutExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("rule-chain-transaction")); + executeOnTimeout(); + } + + @PreDestroy + public void destroy() { + if (timeoutExecutor != null) { + timeoutExecutor.shutdownNow(); + } + } + + @Override + public void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure) { + transactionLock.lock(); + try { + BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> + new LinkedBlockingQueue<>(finalQueueSize)); + + TbTransactionTask transactionTask = new TbTransactionTask(msg, onStart, onEnd, onFailure, System.currentTimeMillis() + duration); + int queueSize = queue.size(); + if (queueSize >= finalQueueSize) { + log.trace("Queue has no space: {}", transactionTask); + executeOnFailure(transactionTask.getOnFailure(), "Queue has no space!"); + } else { + addMsgToQueues(queue, transactionTask); + if (queueSize == 0) { + executeOnSuccess(transactionTask.getOnStart(), transactionTask.getMsg()); + } else { + log.trace("Msg [{}][{}] is waiting to start transaction!", msg.getId(), msg.getType()); + } + } + } finally { + transactionLock.unlock(); + } + } + + @Override + public void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { + Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); + if (address.isPresent()) { + sendTransactionEventToRemoteServer(msg, address.get()); + executeOnSuccess(onSuccess, msg); + } else { + endLocalTransaction(msg, onSuccess, onFailure); + } + } + + @Override + public void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] data) { + endLocalTransaction(TbMsg.fromBytes(data), msg -> { + }, error -> { + }); + } + + private void addMsgToQueues(BlockingQueue queue, TbTransactionTask transactionTask) { + queue.offer(transactionTask); + timeoutQueue.offer(transactionTask); + log.trace("Added msg to queue, size: [{}]", queue.size()); + } + + private void endLocalTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { + transactionLock.lock(); + try { + BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> + new LinkedBlockingQueue<>(finalQueueSize)); + + TbTransactionTask currentTransactionTask = queue.peek(); + if (currentTransactionTask != null) { + if (currentTransactionTask.getMsg().getTransactionData().getTransactionId().equals(msg.getTransactionData().getTransactionId())) { + currentTransactionTask.setCompleted(true); + queue.poll(); + log.trace("Removed msg from queue, size [{}]", queue.size()); + + executeOnSuccess(currentTransactionTask.getOnEnd(), currentTransactionTask.getMsg()); + executeOnSuccess(onSuccess, msg); + + TbTransactionTask nextTransactionTask = queue.peek(); + if (nextTransactionTask != null) { + executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); + } + } else { + log.trace("Task has expired!"); + executeOnFailure(onFailure, "Task has expired!"); + } + } else { + log.trace("Queue is empty, previous task has expired!"); + executeOnFailure(onFailure, "Queue is empty, previous task has expired!"); + } + } finally { + transactionLock.unlock(); + } + } + + private void executeOnTimeout() { + timeoutExecutor.submit(() -> { + while (true) { + TbTransactionTask transactionTask = timeoutQueue.peek(); + if (transactionTask != null) { + long sleepDuration = 0L; + transactionLock.lock(); + try { + if (transactionTask.isCompleted()) { + timeoutQueue.poll(); + } else { + long expIn = transactionTask.getExpirationTime() - System.currentTimeMillis(); + if (expIn < 0) { + log.trace("Task has expired! Deleting it...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); + timeoutQueue.poll(); + executeOnFailure(transactionTask.getOnFailure(), "Task has expired!"); + + BlockingQueue queue = transactionMap.get(transactionTask.getMsg().getTransactionData().getOriginatorId()); + if (queue != null) { + queue.poll(); + TbTransactionTask nextTransactionTask = queue.peek(); + if (nextTransactionTask != null) { + executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); + } + } + } else { + sleepDuration = Math.min(expIn, duration); + } + } + } finally { + transactionLock.unlock(); + } + if (sleepDuration > 0L) { + try { + log.trace("Task has not expired! Continue executing...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); + TimeUnit.MILLISECONDS.sleep(sleepDuration); + } catch (InterruptedException e) { + throw new IllegalStateException("Thread interrupted", e); + } + } + } else { + try { + log.trace("Queue is empty, waiting for tasks!"); + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + throw new IllegalStateException("Thread interrupted", e); + } + } + } + }); + } + + private void executeOnFailure(Consumer onFailure, String exception) { + executeCallback(() -> { + onFailure.accept(new RuntimeException(exception)); + return null; + }); + } + + private void executeOnSuccess(Consumer onSuccess, TbMsg tbMsg) { + executeCallback(() -> { + onSuccess.accept(tbMsg); + return null; + }); + } + + private void executeCallback(Callable task) { + callbackExecutor.executeAsync(task); + } + + private void sendTransactionEventToRemoteServer(TbMsg msg, ServerAddress address) { + log.trace("[{}][{}] Originator is monitored on other server: {}", msg.getTransactionData().getOriginatorId(), msg.getTransactionData().getTransactionId(), address); + clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java b/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java new file mode 100644 index 0000000000..c86f882ff7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transaction; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.msg.TbMsg; + +import java.util.function.Consumer; + +@Data +@AllArgsConstructor +public final class TbTransactionTask { + + private final TbMsg msg; + private final Consumer onStart; + private final Consumer onEnd; + private final Consumer onFailure; + private final long expirationTime; + + private boolean isCompleted; + + public TbTransactionTask(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure, long expirationTime) { + this.msg = msg; + this.onStart = onStart; + this.onEnd = onEnd; + this.onFailure = onFailure; + this.expirationTime = expirationTime; + this.isCompleted = false; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java deleted file mode 100644 index c9e37e6791..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import java.util.UUID; -import java.util.function.Consumer; - -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; - -@Slf4j -@Service -@TbCoreComponent -public class DefaultTbCoreToTransportService implements TbCoreToTransportService { - - private final PartitionService partitionService; - private final TbQueueProducer> tbTransportProducer; - - public DefaultTbCoreToTransportService(PartitionService partitionService, TbQueueProducerProvider tbQueueProducerProvider) { - this.partitionService = partitionService; - this.tbTransportProducer = tbQueueProducerProvider.getTransportNotificationsMsgProducer(); - } - - @Override - public void process(String nodeId, ToTransportMsg msg) { - process(nodeId, msg, null, null); - } - - @Override - public void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, nodeId); - UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); - log.trace("[{}][{}] Pushing session data to topic: {}", tpi.getFullTopicName(), sessionId, msg); - TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, msg); - tbTransportProducer.send(tpi, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); - } - - private static class QueueCallbackAdaptor implements TbQueueCallback { - private final Runnable onSuccess; - private final Consumer onFailure; - - QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { - this.onSuccess = onSuccess; - this.onFailure = onFailure; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (onSuccess != null) { - onSuccess.run(); - } - } - - @Override - public void onFailure(Throwable t) { - if (onFailure != null) { - onFailure.accept(t); - } - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java rename to application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java index 7456ca7662..8e820fbffe 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java @@ -25,7 +25,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -34,19 +33,14 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; @@ -58,15 +52,10 @@ import java.util.concurrent.locks.ReentrantLock; */ @Slf4j @Service -@TbCoreComponent -public class DefaultTransportApiService implements TransportApiService { +public class LocalTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); - //TODO: Constructor dependencies; - @Autowired - private TenantService tenantService; - @Autowired private DeviceService deviceService; @@ -85,20 +74,17 @@ public class DefaultTransportApiService implements TransportApiService { private ReentrantLock deviceCreationLock = new ReentrantLock(); @Override - public ListenableFuture> handle(TbProtoQueueMsg tbProtoQueueMsg) { - TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue(); + public ListenableFuture handle(TransportApiRequestMsg transportApiRequestMsg) { if (transportApiRequestMsg.hasValidateTokenRequestMsg()) { ValidateDeviceTokenRequestMsg msg = transportApiRequestMsg.getValidateTokenRequestMsg(); - return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + return validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN); } else if (transportApiRequestMsg.hasValidateX509CertRequestMsg()) { ValidateDeviceX509CertRequestMsg msg = transportApiRequestMsg.getValidateX509CertRequestMsg(); - return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + return validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE); } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); - } else if (transportApiRequestMsg.hasGetTenantRoutingInfoRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getGetTenantRoutingInfoRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + return handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()); } - return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + return getEmptyTransportApiResponseFuture(); } private ListenableFuture validateCredentials(String credentialsId, DeviceCredentialsType credentialsType) { @@ -139,13 +125,6 @@ public class DefaultTransportApiService implements TransportApiService { }, dbCallbackExecutorService); } - private ListenableFuture handle(GetTenantRoutingInfoRequestMsg requestMsg) { - TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB())); - ListenableFuture tenantFuture = tenantService.findTenantByIdAsync(TenantId.SYS_TENANT_ID, tenantId); - return Futures.transform(tenantFuture, tenant -> TransportApiResponseMsg.newBuilder() - .setGetTenantRoutingInfoResponseMsg(GetTenantRoutingInfoResponseMsg.newBuilder().setIsolatedTbCore(tenant.isIsolatedTbCore()) - .setIsolatedTbRuleEngine(tenant.isIsolatedTbRuleEngine()).build()).build(), dbCallbackExecutorService); - } private ListenableFuture getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { return Futures.transform(deviceService.findDeviceByIdAsync(TenantId.SYS_TENANT_ID, deviceId), device -> { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java new file mode 100644 index 0000000000..32ff77877d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java @@ -0,0 +1,227 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import akka.actor.ActorRef; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.DonAsynchron; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.common.transport.service.AbstractTransportService; +import org.thingsboard.server.dao.device.ClaimDevicesService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 12.10.18. + */ +@Slf4j +@Service +@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "local") +public class LocalTransportService extends AbstractTransportService implements RuleEngineTransportService { + + @Autowired + private TransportApiService transportApiService; + + @Autowired + private ActorSystemContext actorContext; + + //TODO: completely replace this routing with the Kafka routing by partition ids. + @Autowired + private ClusterRoutingService routingService; + @Autowired + private ClusterRpcService rpcService; + @Autowired + private DataDecodingEncodingService encodingService; + @Autowired + private ClaimDevicesService claimDevicesService; + + @PostConstruct + public void init() { + super.init(); + } + + @PreDestroy + public void destroy() { + super.destroy(); + } + + @Override + public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { + DonAsynchron.withCallback( + transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), + transportApiResponseMsg -> { + if (callback != null) { + callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); + } + }, + getThrowableConsumer(callback), transportCallbackExecutor); + } + + @Override + public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { + DonAsynchron.withCallback( + transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), + transportApiResponseMsg -> { + if (callback != null) { + callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); + } + }, + getThrowableConsumer(callback), transportCallbackExecutor); + } + + @Override + public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { + DonAsynchron.withCallback( + transportApiService.handle(TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), + transportApiResponseMsg -> { + if (callback != null) { + callback.onSuccess(transportApiResponseMsg.getGetOrCreateDeviceResponseMsg()); + } + }, + getThrowableConsumer(callback), transportCallbackExecutor); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSessionEvent(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostTelemetry(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostAttributes(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setGetAttributes(msg).build(), callback); + } + + @Override + public void process(SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscriptionInfo(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToAttributes(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToRPC(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToDeviceRPCCallResponse(msg).build(), callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToServerRPCCallRequest(msg).build(), callback); + } + + @Override + protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { + TransportToDeviceActorMsg toDeviceActorMsg = TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setClaimDevice(msg).build(); + + TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); + Optional address = routingService.resolveById(wrapper.getDeviceId()); + if (address.isPresent()) { + rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); + callback.onSuccess(null); + } else { + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(msg.getDeviceIdMSB(), msg.getDeviceIdLSB())); + DonAsynchron.withCallback(claimDevicesService.registerClaimingInfo(tenantId, deviceId, msg.getSecretKey(), msg.getDurationMs()), + callback::onSuccess, callback::onError); + } + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg) { + process(nodeId, msg, null, null); + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { + processToTransportMsg(msg); + if (onSuccess != null) { + onSuccess.run(); + } + } + + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { + TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); + Optional address = routingService.resolveById(wrapper.getDeviceId()); + if (address.isPresent()) { + rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); + } else { + actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); + } + if (callback != null) { + callback.onSuccess(null); + } + } + + private Consumer getThrowableConsumer(TransportServiceCallback callback) { + return e -> { + if (callback != null) { + callback.onError(e); + } + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java new file mode 100644 index 0000000000..7aa438c64f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java @@ -0,0 +1,251 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import akka.actor.ActorRef; +import io.github.bucket4j.Bandwidth; +import io.github.bucket4j.BlockingBucket; +import io.github.bucket4j.Bucket4j; +import io.github.bucket4j.local.LocalBucket; +import io.github.bucket4j.local.LocalBucketBuilder; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; +import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.time.Duration; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 09.10.18. + */ +@Slf4j +@Service +@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") +public class RemoteRuleEngineTransportService implements RuleEngineTransportService { + + @Value("${transport.remote.rule_engine.topic}") + private String ruleEngineTopic; + @Value("${transport.remote.notifications.topic}") + private String notificationsTopic; + @Value("${transport.remote.rule_engine.poll_interval}") + private int pollDuration; + @Value("${transport.remote.rule_engine.auto_commit_interval}") + private int autoCommitInterval; + + @Value("${transport.remote.rule_engine.poll_records_pack_size}") + private int pollRecordsPackSize; + @Value("${transport.remote.rule_engine.max_poll_records_per_second}") + private long pollRecordsPerSecond; + @Value("${transport.remote.rule_engine.max_poll_records_per_minute}") + private long pollRecordsPerMinute; + @Value("${transport.remote.rule_engine.stats.enabled:false}") + private boolean statsEnabled; + + @Autowired + private TbKafkaSettings kafkaSettings; + + @Autowired + private TbNodeIdProvider nodeIdProvider; + + @Autowired + private ActorSystemContext actorContext; + + //TODO: completely replace this routing with the Kafka routing by partition ids. + @Autowired + private ClusterRoutingService routingService; + @Autowired + private ClusterRpcService rpcService; + @Autowired + private DataDecodingEncodingService encodingService; + + private TBKafkaConsumerTemplate ruleEngineConsumer; + private TBKafkaProducerTemplate notificationsProducer; + + private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-main-consumer")); + + private volatile boolean stopped = false; + + private final RuleEngineStats stats = new RuleEngineStats(); + + @PostConstruct + public void init() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder notificationsProducerBuilder = TBKafkaProducerTemplate.builder(); + notificationsProducerBuilder.settings(kafkaSettings); + notificationsProducerBuilder.clientId("producer-transport-notification-" + nodeIdProvider.getNodeId()); + notificationsProducerBuilder.encoder(new ToTransportMsgEncoder()); + + notificationsProducer = notificationsProducerBuilder.build(); + notificationsProducer.init(); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder ruleEngineConsumerBuilder = TBKafkaConsumerTemplate.builder(); + ruleEngineConsumerBuilder.settings(kafkaSettings); + ruleEngineConsumerBuilder.topic(ruleEngineTopic); + ruleEngineConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); + ruleEngineConsumerBuilder.groupId("tb-node"); + ruleEngineConsumerBuilder.autoCommit(true); + ruleEngineConsumerBuilder.autoCommitIntervalMs(autoCommitInterval); + ruleEngineConsumerBuilder.maxPollRecords(pollRecordsPackSize); + ruleEngineConsumerBuilder.decoder(new ToRuleEngineMsgDecoder()); + + ruleEngineConsumer = ruleEngineConsumerBuilder.build(); + ruleEngineConsumer.subscribe(); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + log.info("Received application ready event. Starting polling for events."); + LocalBucketBuilder builder = Bucket4j.builder(); + builder.addLimit(Bandwidth.simple(pollRecordsPerSecond, Duration.ofSeconds(1))); + builder.addLimit(Bandwidth.simple(pollRecordsPerMinute, Duration.ofMinutes(1))); + LocalBucket pollRateBucket = builder.build(); + BlockingBucket blockingPollRateBucket = pollRateBucket.asScheduler(); + + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + ConsumerRecords records = ruleEngineConsumer.poll(Duration.ofMillis(pollDuration)); + int recordsCount = records.count(); + if (recordsCount > 0) { + while (!blockingPollRateBucket.tryConsume(recordsCount, TimeUnit.SECONDS.toNanos(5))) { + log.info("Rule Engine consumer is busy. Required tokens: [{}]. Available tokens: [{}].", recordsCount, pollRateBucket.getAvailableTokens()); + Thread.sleep(TimeUnit.SECONDS.toMillis(1)); + } + log.trace("Processing {} records", recordsCount); + } + records.forEach(record -> { + try { + ToRuleEngineMsg toRuleEngineMsg = ruleEngineConsumer.decode(record); + log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); + if (toRuleEngineMsg.hasToDeviceActorMsg()) { + forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg()); + } + } catch (Throwable e) { + log.warn("Failed to process the notification.", e); + } + }); + } catch (Exception e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + @Scheduled(fixedDelayString = "${transport.remote.rule_engine.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + stats.printStats(); + } + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg) { + process(nodeId, msg, null, null); + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { + String topic = notificationsTopic + "." + nodeId; + UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); + ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); + log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); + notificationsProducer.send(topic, sessionId.toString(), transportMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + } + + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg) { + if (statsEnabled) { + stats.log(toDeviceActorMsg); + } + TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); + Optional address = routingService.resolveById(wrapper.getDeviceId()); + if (address.isPresent()) { + log.trace("[{}] Pushing message to remote server: {}", address.get(), toDeviceActorMsg); + rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); + } else { + log.trace("Pushing message to local server: {}", toDeviceActorMsg); + actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); + } + } + + @PreDestroy + public void destroy() { + stopped = true; + if (ruleEngineConsumer != null) { + ruleEngineConsumer.unsubscribe(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + } + + private static class QueueCallbackAdaptor implements Callback { + private final Runnable onSuccess; + private final Consumer onFailure; + + QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception == null) { + if (onSuccess != null) { + onSuccess.run(); + } + } else { + if (onFailure != null) { + onFailure.accept(exception); + } + } + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java new file mode 100644 index 0000000000..0c9efc7d1c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -0,0 +1,111 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.*; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.concurrent.*; + +/** + * Created by ashvayka on 05.10.18. + */ +@Slf4j +@Component +@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") +public class RemoteTransportApiService { + + @Value("${transport.remote.transport_api.requests_topic}") + private String transportApiRequestsTopic; + @Value("${transport.remote.transport_api.max_pending_requests}") + private int maxPendingRequests; + @Value("${transport.remote.transport_api.request_timeout}") + private long requestTimeout; + @Value("${transport.remote.transport_api.request_poll_interval}") + private int responsePollDuration; + @Value("${transport.remote.transport_api.request_auto_commit_interval}") + private int autoCommitInterval; + + @Autowired + private TbKafkaSettings kafkaSettings; + + @Autowired + private TbNodeIdProvider nodeIdProvider; + + @Autowired + private TransportApiService transportApiService; + + private ExecutorService transportCallbackExecutor; + + private TbKafkaResponseTemplate transportApiTemplate; + + @PostConstruct + public void init() { + this.transportCallbackExecutor = Executors.newWorkStealingPool(100); + + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder responseBuilder = TBKafkaProducerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.clientId("producer-transport-api-response-" + nodeIdProvider.getNodeId()); + responseBuilder.encoder(new TransportApiResponseEncoder()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder requestBuilder = TBKafkaConsumerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.topic(transportApiRequestsTopic); + requestBuilder.clientId(nodeIdProvider.getNodeId()); + requestBuilder.groupId("tb-node"); + requestBuilder.autoCommit(true); + requestBuilder.autoCommitIntervalMs(autoCommitInterval); + requestBuilder.decoder(new TransportApiRequestDecoder()); + + TbKafkaResponseTemplate.TbKafkaResponseTemplateBuilder + builder = TbKafkaResponseTemplate.builder(); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(maxPendingRequests); + builder.requestTimeout(requestTimeout); + builder.pollInterval(responsePollDuration); + builder.executor(transportCallbackExecutor); + builder.handler(transportApiService); + transportApiTemplate = builder.build(); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + log.info("Received application ready event. Starting polling for events."); + transportApiTemplate.init(); + } + + @PreDestroy + public void destroy() { + if (transportApiTemplate != null) { + transportApiTemplate.stop(); + } + if (transportCallbackExecutor != null) { + transportCallbackExecutor.shutdownNow(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java similarity index 63% rename from application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java rename to application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java index c362c5e889..dea6e6d33c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue; +package org.thingsboard.server.service.transport; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; @@ -21,26 +21,31 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -public class TbCoreConsumerStats { +public class RuleEngineStats { private final AtomicInteger totalCounter = new AtomicInteger(0); private final AtomicInteger sessionEventCounter = new AtomicInteger(0); + private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); + private final AtomicInteger postAttributesCounter = new AtomicInteger(0); private final AtomicInteger getAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); + private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); - private final AtomicInteger deviceStateCounter = new AtomicInteger(0); - private final AtomicInteger subscriptionMsgCounter = new AtomicInteger(0); - private final AtomicInteger toCoreNotificationsCounter = new AtomicInteger(0); - public void log(TransportProtos.TransportToDeviceActorMsg msg) { totalCounter.incrementAndGet(); if (msg.hasSessionEvent()) { sessionEventCounter.incrementAndGet(); } + if (msg.hasPostTelemetry()) { + postTelemetryCounter.incrementAndGet(); + } + if (msg.hasPostAttributes()) { + postAttributesCounter.incrementAndGet(); + } if (msg.hasGetAttributes()) { getAttributesCounter.incrementAndGet(); } @@ -53,6 +58,9 @@ public class TbCoreConsumerStats { if (msg.hasToDeviceRPCCallResponse()) { toDeviceRPCCallResponseCounter.incrementAndGet(); } + if (msg.hasToServerRPCCallRequest()) { + toServerRPCCallRequestCounter.incrementAndGet(); + } if (msg.hasSubscriptionInfo()) { subscriptionInfoCounter.incrementAndGet(); } @@ -61,32 +69,15 @@ public class TbCoreConsumerStats { } } - public void log(TransportProtos.DeviceStateServiceMsgProto msg) { - totalCounter.incrementAndGet(); - deviceStateCounter.incrementAndGet(); - } - - public void log(TransportProtos.SubscriptionMgrMsgProto msg) { - totalCounter.incrementAndGet(); - subscriptionMsgCounter.incrementAndGet(); - } - - public void log(TransportProtos.ToCoreNotificationMsg msg) { - totalCounter.incrementAndGet(); - toCoreNotificationsCounter.incrementAndGet(); - } - public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { - log.info("Total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + - " deviceState [{}] subMgr [{}] coreNfs [{}]", - total, sessionEventCounter.getAndSet(0), - getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), + log.info("Transport total [{}] sessionEvents [{}] telemetry [{}] attributes [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] " + + "toServerRpc [{}] subInfo [{}] claimDevice [{}] ", + total, sessionEventCounter.getAndSet(0), postTelemetryCounter.getAndSet(0), + postAttributesCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), - subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0) - , deviceStateCounter.getAndSet(0), subscriptionMsgCounter.getAndSet(0), toCoreNotificationsCounter.getAndSet(0)); + toServerRPCCallRequestCounter.getAndSet(0), subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); } } - } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java new file mode 100644 index 0000000000..0d401e4d9c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; + +import java.util.function.Consumer; + +/** + * Created by ashvayka on 05.10.18. + */ +public interface RuleEngineTransportService { + + void process(String nodeId, DeviceActorToTransportMsg msg); + + void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java deleted file mode 100644 index 566bf99cd3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueResponseTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.provider.TbCoreQueueFactory; -import org.thingsboard.server.queue.util.TbCoreComponent; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.concurrent.*; - -/** - * Created by ashvayka on 05.10.18. - */ -@Slf4j -@Service -@TbCoreComponent -public class TbCoreTransportApiService { - - private final TbCoreQueueFactory tbCoreQueueFactory; - private final TransportApiService transportApiService; - - @Value("${queue.transport_api.max_pending_requests:10000}") - private int maxPendingRequests; - @Value("${queue.transport_api.max_requests_timeout:10000}") - private long requestTimeout; - @Value("${queue.transport_api.request_poll_interval:25}") - private int responsePollDuration; - @Value("${queue.transport_api.max_callback_threads:100}") - private int maxCallbackThreads; - - private ExecutorService transportCallbackExecutor; - private TbQueueResponseTemplate, - TbProtoQueueMsg> transportApiTemplate; - - public TbCoreTransportApiService(TbCoreQueueFactory tbCoreQueueFactory, TransportApiService transportApiService) { - this.tbCoreQueueFactory = tbCoreQueueFactory; - this.transportApiService = transportApiService; - } - - @PostConstruct - public void init() { - this.transportCallbackExecutor = Executors.newWorkStealingPool(maxCallbackThreads); - TbQueueProducer> producer = tbCoreQueueFactory.createTransportApiResponseProducer(); - TbQueueConsumer> consumer = tbCoreQueueFactory.createTransportApiRequestConsumer(); - - DefaultTbQueueResponseTemplate.DefaultTbQueueResponseTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueResponseTemplate.builder(); - builder.requestTemplate(consumer); - builder.responseTemplate(producer); - builder.maxPendingRequests(maxPendingRequests); - builder.requestTimeout(requestTimeout); - builder.pollInterval(responsePollDuration); - builder.executor(transportCallbackExecutor); - builder.handler(transportApiService); - transportApiTemplate = builder.build(); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting polling for events."); - transportApiTemplate.init(transportApiService); - } - - @PreDestroy - public void destroy() { - if (transportApiTemplate != null) { - transportApiTemplate.stop(); - } - if (transportCallbackExecutor != null) { - transportCallbackExecutor.shutdownNow(); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java similarity index 64% rename from application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java rename to application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java index 2b2c203ec5..9f08463f91 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.processing; +package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.kafka.TbKafkaDecoder; -import java.util.UUID; +import java.io.IOException; -public class IdMsgPair { - final UUID uuid; - final TbProtoQueueMsg msg; - - public IdMsgPair(UUID uuid, TbProtoQueueMsg msg) { - this.uuid = uuid; - this.msg = msg; +/** + * Created by ashvayka on 05.10.18. + */ +public class ToRuleEngineMsgDecoder implements TbKafkaDecoder { + @Override + public ToRuleEngineMsg decode(byte[] data) throws IOException { + return ToRuleEngineMsg.parseFrom(data); } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java rename to application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java index e0d5e2121a..9c3f5634bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java @@ -16,13 +16,14 @@ package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.kafka.TbKafkaEncoder; -import java.util.function.Consumer; - -public interface TbCoreToTransportService { - - void process(String nodeId, ToTransportMsg msg); - - void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure); - +/** + * Created by ashvayka on 05.10.18. + */ +public class ToTransportMsgEncoder implements TbKafkaEncoder { + @Override + public byte[] encode(ToTransportMsg value) { + return value.toByteArray(); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java new file mode 100644 index 0000000000..ac1c5965f3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.kafka.TbKafkaDecoder; + +import java.io.IOException; + +/** + * Created by ashvayka on 05.10.18. + */ +public class TransportApiRequestDecoder implements TbKafkaDecoder { + @Override + public TransportApiRequestMsg decode(byte[] data) throws IOException { + return TransportApiRequestMsg.parseFrom(data); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java new file mode 100644 index 0000000000..35f8a7b1aa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import org.thingsboard.server.kafka.TbKafkaEncoder; + +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; + +/** + * Created by ashvayka on 05.10.18. + */ +public class TransportApiResponseEncoder implements TbKafkaEncoder { + @Override + public byte[] encode(TransportApiResponseMsg value) { + return value.toByteArray(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java index c9edfa9331..2964934313 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java @@ -15,13 +15,11 @@ */ package org.thingsboard.server.service.transport; -import org.thingsboard.server.queue.TbQueueHandler; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.kafka.TbKafkaHandler; /** * Created by ashvayka on 05.10.18. */ -public interface TransportApiService extends TbQueueHandler, TbProtoQueueMsg> { +public interface TransportApiService extends TbKafkaHandler { } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java index ba4e7baab1..34abac72bb 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.common.msg.queue.TbCallback; import java.io.Serializable; import java.util.UUID; @@ -37,11 +36,9 @@ public class TransportToDeviceActorMsgWrapper implements TbActorMsg, DeviceAware private final TenantId tenantId; private final DeviceId deviceId; private final TransportToDeviceActorMsg msg; - private final TbCallback callback; - public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg, TbCallback callback) { + public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg) { this.msg = msg; - this.callback = callback; this.tenantId = new TenantId(new UUID(msg.getSessionInfo().getTenantIdMSB(), msg.getSessionInfo().getTenantIdLSB())); this.deviceId = new DeviceId(new UUID(msg.getSessionInfo().getDeviceIdMSB(), msg.getSessionInfo().getDeviceIdLSB())); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java index 61d81ae0b0..2399450d86 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java @@ -17,27 +17,47 @@ package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.thingsboard.server.dao.util.PsqlDao; +import org.springframework.scheduling.annotation.Scheduled; +import org.thingsboard.server.dao.util.PsqlTsAnyDao; import java.sql.Connection; +import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; - +@PsqlTsAnyDao @Slf4j -@PsqlDao -public abstract class AbstractCleanUpService { +public abstract class AbstractTimeseriesCleanUpService { + + @Value("${sql.ttl.ts_key_value_ttl}") + protected long systemTtl; + + @Value("${sql.ttl.enabled}") + private boolean ttlTaskExecutionEnabled; @Value("${spring.datasource.url}") - protected String dbUrl; + private String dbUrl; @Value("${spring.datasource.username}") - protected String dbUserName; + private String dbUserName; @Value("${spring.datasource.password}") - protected String dbPassword; + private String dbPassword; + + @Scheduled(initialDelayString = "${sql.ttl.execution_interval_ms}", fixedDelayString = "${sql.ttl.execution_interval_ms}") + public void cleanUp() { + if (ttlTaskExecutionEnabled) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + doCleanUp(conn); + } catch (SQLException e) { + log.error("SQLException occurred during TTL task execution ", e); + } + } + } + + protected abstract void doCleanUp(Connection connection); protected long executeQuery(Connection conn, String query) { long removed = 0L; @@ -54,7 +74,7 @@ public abstract class AbstractCleanUpService { return removed; } - protected void getWarnings(Statement statement) throws SQLException { + private void getWarnings(Statement statement) throws SQLException { SQLWarning warnings = statement.getWarnings(); if (warnings != null) { log.debug("{}", warnings.getMessage()); @@ -66,6 +86,4 @@ public abstract class AbstractCleanUpService { } } - protected abstract void doCleanUp(Connection connection); - -} +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java index cd403ee3b8..ab344dc518 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl.timeseries; +package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java index f5898b9b20..1dbdb4ad55 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl.timeseries; +package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java deleted file mode 100644 index a608ca257b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.ttl.events; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.service.ttl.AbstractCleanUpService; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -@PsqlDao -@Slf4j -@Service -public class EventsCleanUpService extends AbstractCleanUpService { - - @Value("${sql.ttl.events.events_ttl}") - private long ttl; - - @Value("${sql.ttl.events.debug_events_ttl}") - private long debugTtl; - - @Value("${sql.ttl.events.enabled}") - private boolean ttlTaskExecutionEnabled; - - @Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}") - public void cleanUp() { - if (ttlTaskExecutionEnabled) { - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - doCleanUp(conn); - } catch (SQLException e) { - log.error("SQLException occurred during TTL task execution ", e); - } - } - } - - @Override - protected void doCleanUp(Connection connection) { - long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);"); - log.info("Total events removed by TTL: [{}]", totalEventsRemoved); - } -} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java deleted file mode 100644 index 75b07b9176..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.ttl.timeseries; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.thingsboard.server.dao.util.PsqlTsAnyDao; -import org.thingsboard.server.service.ttl.AbstractCleanUpService; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -@PsqlTsAnyDao -@Slf4j -public abstract class AbstractTimeseriesCleanUpService extends AbstractCleanUpService { - - @Value("${sql.ttl.ts.ts_key_value_ttl}") - protected long systemTtl; - - @Value("${sql.ttl.ts.enabled}") - private boolean ttlTaskExecutionEnabled; - - @Scheduled(initialDelayString = "${sql.ttl.ts.execution_interval_ms}", fixedDelayString = "${sql.ttl.ts.execution_interval_ms}") - public void cleanUp() { - if (ttlTaskExecutionEnabled) { - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - doCleanUp(conn); - } catch (SQLException e) { - log.error("SQLException occurred during TTL task execution ", e); - } - } - } - -} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java index 4b7f0f6366..1b8173c424 100644 --- a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java @@ -24,7 +24,6 @@ import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.UpdateMessage; -import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -39,7 +38,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @Service -@TbCoreComponent @Slf4j public class DefaultUpdateService implements UpdateService { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java b/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java similarity index 97% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java rename to application/src/main/java/org/thingsboard/server/utils/JsonUtils.java index 583b73dba1..5621362c09 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.utils; import com.google.gson.JsonElement; import com.google.gson.JsonObject; diff --git a/application/src/main/proto/cluster.proto b/application/src/main/proto/cluster.proto new file mode 100644 index 0000000000..cfacc66121 --- /dev/null +++ b/application/src/main/proto/cluster.proto @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; +package cluster; + +option java_package = "org.thingsboard.server.gen.cluster"; +option java_outer_classname = "ClusterAPIProtos"; + +service ClusterRpcService { + rpc handleMsgs(stream ClusterMessage) returns (stream ClusterMessage) {} +} + +message ClusterMessage { + MessageType messageType = 1; + MessageMataInfo messageMetaInfo = 2; + ServerAddress serverAddress = 3; + bytes payload = 4; +} + +message ServerAddress { + string host = 1; + int32 port = 2; +} + +message MessageMataInfo { + string payloadMetaInfo = 1; + repeated string tags = 2; +} + +enum MessageType { + + //Cluster control messages + RPC_SESSION_CREATE_REQUEST_MSG = 0; + TO_ALL_NODES_MSG = 1; + RPC_SESSION_TELL_MSG = 2; + RPC_BROADCAST_MSG = 3; + CONNECT_RPC_MESSAGE =4; + + CLUSTER_ACTOR_MESSAGE = 5; + // Messages related to TelemetrySubscriptionService + CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE = 6; + CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE = 7; + CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE = 8; + CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE = 9; + CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE = 10; + CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE = 11; + CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE = 12; + + CLUSTER_DEVICE_STATE_SERVICE_MESSAGE = 13; + CLUSTER_TRANSACTION_SERVICE_MESSAGE = 14; +} + +// Messages related to CLUSTER_TELEMETRY_MESSAGE +message SubscriptionProto { + string sessionId = 1; + int32 subscriptionId = 2; + string entityType = 3; + string tenantId = 4; + string entityId = 5; + string type = 6; + bool allKeys = 7; + repeated SubscriptionKetStateProto keyStates = 8; + string scope = 9; +} + +message SubscriptionUpdateProto { + string sessionId = 1; + int32 subscriptionId = 2; + int32 errorCode = 3; + string errorMsg = 4; + repeated SubscriptionUpdateValueListProto data = 5; +} + +message AttributeUpdateProto { + string entityType = 1; + string entityId = 2; + string scope = 3; + repeated KeyValueProto data = 4; +} + +message TimeseriesUpdateProto { + string entityType = 1; + string entityId = 2; + repeated KeyValueProto data = 4; +} + +message SessionCloseProto { + string sessionId = 1; +} + +message SubscriptionCloseProto { + string sessionId = 1; + int32 subscriptionId = 2; +} + +message SubscriptionKetStateProto { + string key = 1; + int64 ts = 2; +} + +message SubscriptionUpdateValueListProto { + string key = 1; + repeated int64 ts = 2; + repeated string value = 3; +} + +message KeyValueProto { + string key = 1; + int64 ts = 2; + int32 valueType = 3; + string strValue = 4; + int64 longValue = 5; + double doubleValue = 6; + bool boolValue = 7; + string jsonValue = 8; +} + +message FromDeviceRPCResponseProto { + int64 requestIdMSB = 1; + int64 requestIdLSB = 2; + string response = 3; + int32 error = 4; +} + +message DeviceStateServiceMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; +} diff --git a/common/queue/src/main/proto/jsinvoke.proto b/application/src/main/proto/jsinvoke.proto similarity index 100% rename from common/queue/src/main/proto/jsinvoke.proto rename to application/src/main/proto/jsinvoke.proto diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 614f85ba0e..e147325f42 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -28,9 +28,6 @@ - - - diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index c896b3fc85..8dc7089bbe 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -31,7 +31,7 @@ server: key-store-type: "${SSL_KEY_STORE_TYPE:PKCS12}" # Alias that identifies the key in the key store key-alias: "${SSL_KEY_ALIAS:tomcat}" - log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:false}" + log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:true}" ws: send_timeout: "${TB_SERVER_WS_SEND_TIMEOUT:5000}" limits: @@ -70,7 +70,21 @@ zk: # Name of the directory in zookeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" +# RPC connection parameters. Used only in cluster mode only. +rpc: + bind_host: "${RPC_HOST:localhost}" + bind_port: "${RPC_PORT:9001}" + +# Clustering properties related to consistent-hashing. See architecture docs for more details. cluster: + # Unique id for this node (autogenerated if empty) + node_id: "${CLUSTER_NODE_ID:}" + # Name of hash function used for consistent hash ring. + hash_function_name: "${CLUSTER_HASH_FUNCTION_NAME:murmur3_128}" + # Amount of virtual nodes in consistent hash ring. + vitrual_nodes_size: "${CLUSTER_VIRTUAL_NODES_SIZE:16}" + # Queue partition id for current node + partition_id: "${QUEUE_PARTITION_ID:0}" stats: enabled: "${TB_CLUSTER_STATS_ENABLED:false}" print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" @@ -201,18 +215,14 @@ sql: # Specify Interval size for new data chunks storage. chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" ttl: - ts: - enabled: "${SQL_TTL_TS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day - ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds - events: - enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day - events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds - debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week + enabled: "${SQL_TTL_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds + ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds # Actor system parameters actors: + cluster: + grpc_callback_thread_pool_size: "${ACTORS_CLUSTER_GRPC_CALLBACK_THREAD_POOL_SIZE:10}" tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" session: @@ -255,6 +265,8 @@ actors: enabled: "${ACTORS_QUEUE_ENABLED:true}" # Maximum allowed timeout for persistence into the queue timeout: "${ACTORS_QUEUE_PERSISTENCE_TIMEOUT:30000}" + client_side_rpc: + timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" cache: # caffeine or redis @@ -334,19 +346,19 @@ updates: # spring CORS configuration spring.mvc.cors: - mappings: - # Intercept path - "[/api/**]": - #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. - allowed-origins: "*" - #Comma-separated list of methods to allow. '*' allows all methods. - allowed-methods: "*" - #Comma-separated list of headers to allow in a request. '*' allows all headers. - allowed-headers: "*" - #How long, in seconds, the response from a pre-flight request can be cached by clients. - max-age: "1800" - #Set whether credentials are supported. When not set, credentials are not supported. - allow-credentials: "true" + mappings: + # Intercept path + "[/api/**]": + #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. + allowed-origins: "*" + #Comma-separated list of methods to allow. '*' allows all methods. + allowed-methods: "*" + #Comma-separated list of headers to allow in a request. '*' allows all headers. + allowed-headers: "*" + #How long, in seconds, the response from a pre-flight request can be cached by clients. + max-age: "1800" + #Set whether credentials are supported. When not set, credentials are not supported. + allow-credentials: "true" # spring serve gzip compressed static resources spring.resources.chain: @@ -375,7 +387,7 @@ spring: username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: - maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:5}" + maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:50}" # Audit log parameters audit-log: @@ -415,11 +427,32 @@ audit-log: password: "${AUDIT_LOG_SINK_PASSWORD:}" state: - # Should be greater then transport.sessions.report_timeout - defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}" - defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}" + defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}" + defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" +kafka: + enabled: true + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + transport_api: + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + request_poll_interval: "${TB_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + request_auto_commit_interval: "${TB_TRANSPORT_REQUEST_AUTO_COMMIT_INTERVAL_MS:100}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + js: evaluator: "${JS_EVALUATOR:local}" # local/remote # Built-in JVM JavaScript environment properties @@ -429,27 +462,55 @@ js: # Specify thread pool size for JavaScript sandbox resource monitor monitor_thread_pool_size: "${LOCAL_JS_SANDBOX_MONITOR_THREAD_POOL_SIZE:4}" # Maximum CPU time in milliseconds allowed for script execution - max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:10000}" + max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:3000}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${LOCAL_JS_SANDBOX_MAX_ERRORS:3}" # JS Eval max request timeout. 0 - no timeout max_requests_timeout: "${LOCAL_JS_MAX_REQUEST_TIMEOUT:0}" - # Maximum time in seconds for black listed function to stay in the list. - max_black_list_duration_sec: "${LOCAL_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_LOCAL_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_LOCAL_STATS_PRINT_INTERVAL_MS:10000}" # Remote JavaScript environment properties remote: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${REMOTE_JS_SANDBOX_MAX_ERRORS:3}" - # Maximum time in seconds for black listed function to stay in the list. - max_black_list_duration_sec: "${REMOTE_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_REMOTE_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_REMOTE_STATS_PRINT_INTERVAL_MS:10000}" transport: + type: "${TRANSPORT_TYPE:local}" # local or remote + remote: + transport_api: + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + request_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + request_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + request_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:1000}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" + poll_records_pack_size: "${TB_RULE_ENGINE_MAX_POLL_RECORDS:1000}" + max_poll_records_per_second: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_SECOND:10000}" + max_poll_records_per_minute: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_MINUTE:120000}" + stats: + enabled: "${TB_RULE_ENGINE_STATS_ENABLED:false}" + print_interval_ms: "${TB_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" @@ -462,8 +523,6 @@ transport: type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" - client_side_rpc: - timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" # Local HTTP transport parameters http: enabled: "${HTTP_ENABLED:true}" @@ -507,7 +566,7 @@ swagger: api_path_regex: "${SWAGGER_API_PATH_REGEX:/api.*}" security_path_regex: "${SWAGGER_SECURITY_PATH_REGEX:/api.*}" non_security_path_regex: "${SWAGGER_NON_SECURITY_PATH_REGEX:/api/noauth.*}" - title: "${SWAGGER_TITLE:ThingsBoard REST API}" + title: "${SWAGGER_TITLE:Thingsboard REST API}" description: "${SWAGGER_DESCRIPTION:For instructions how to authorize requests please visit REST API documentation page.}" contact: name: "${SWAGGER_CONTACT_NAME:Thingsboard team}" @@ -517,160 +576,3 @@ swagger: title: "${SWAGGER_LICENSE_TITLE:Apache License Version 2.0}" url: "${SWAGGER_LICENSE_URL:https://github.com/thingsboard/thingsboard/blob/master/LICENSE}" version: "${SWAGGER_VERSION:2.0}" - -queue: - type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) - kafka: - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" - topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" - aws_sqs: - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - queue-properties: - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - pubsub: - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - queue-properties: - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - rabbitmq: - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - partitions: - hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" - virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" - transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" - max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" - request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - core: - topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" - poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - js: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: - - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" - poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" - poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" - topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" - poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - transport: - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - -service: - type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine - # Unique id for this service (autogenerated if empty) - id: "${TB_SERVICE_ID:}" - tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. - diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index 1e445adf3c..bed5689ceb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -33,7 +33,6 @@ import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.junit.runner.RunWith; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootContextLoader; import org.springframework.boot.test.context.SpringBootTest; @@ -116,7 +115,9 @@ public abstract class AbstractControllerTest { */ private static final long DEFAULT_TIMEOUT = -1L; - protected MediaType contentType = MediaType.APPLICATION_JSON; + protected MediaType contentType = new MediaType(MediaType.APPLICATION_JSON.getType(), + MediaType.APPLICATION_JSON.getSubtype(), + Charset.forName("utf8")); protected MockMvc mockMvc; @@ -199,7 +200,6 @@ public abstract class AbstractControllerTest { createUserAndLogin(customerUser, CUSTOMER_USER_PASSWORD); logout(); - log.info("Executed setup"); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java index c74c83dfe8..23e23c31fe 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; import java.util.ArrayList; import java.util.Collections; @@ -72,7 +71,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) + doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) .andExpect(status().isOk()); } @@ -112,27 +111,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { @Test public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); - for (int i = 0; i < 3; i++) { + for (int i=0;i<3;i++) { Asset asset = new Asset(); - asset.setName("My asset B" + i); + asset.setName("My asset B"+i); asset.setType("typeB"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i = 0; i < 7; i++) { + for (int i=0;i<7;i++) { Asset asset = new Asset(); - asset.setName("My asset C" + i); + asset.setName("My asset C"+i); asset.setType("typeC"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i = 0; i < 9; i++) { + for (int i=0;i<9;i++) { Asset asset = new Asset(); - asset.setName("My asset A" + i); + asset.setName("My asset A"+i); asset.setType("typeA"); assets.add(doPost("/api/asset", asset, Asset.class)); } List assetTypes = doGetTyped("/api/asset/types", - new TypeReference>() { - }); + new TypeReference>(){}); Assert.assertNotNull(assetTypes); Assert.assertEquals(3, assetTypes.size()); @@ -148,10 +146,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { asset.setType("default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); - doDelete("/api/asset/" + savedAsset.getId().getId().toString()) + doDelete("/api/asset/"+savedAsset.getId().getId().toString()) .andExpect(status().isOk()); - doGet("/api/asset/" + savedAsset.getId().getId().toString()) + doGet("/api/asset/"+savedAsset.getId().getId().toString()) .andExpect(status().isNotFound()); } @@ -246,16 +244,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) + doDelete("/api/tenant/"+savedTenant2.getId().getId().toString()) .andExpect(status().isOk()); } @Test public void testFindTenantAssets() throws Exception { List assets = new ArrayList<>(); - for (int i = 0; i < 178; i++) { + for (int i=0;i<178;i++) { Asset asset = new Asset(); - asset.setName("Asset" + i); + asset.setName("Asset"+i); asset.setType("default"); assets.add(doPost("/api/asset", asset, Asset.class)); } @@ -271,8 +269,6 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } } while (pageData.hasNext()); - loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE)); - Collections.sort(assets, idComparator); Collections.sort(loadedAssets, idComparator); @@ -283,10 +279,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void testFindTenantAssetsByName() throws Exception { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i = 0; i < 143; i++) { + for (int i=0;i<143;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1 + suffix; + String name = title1+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -294,10 +290,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i = 0; i < 75; i++) { + for (int i=0;i<75;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2 + suffix; + String name = title2+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -338,7 +334,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsTitle2, loadedAssetsTitle2); for (Asset asset : loadedAssetsTitle1) { - doDelete("/api/asset/" + asset.getId().getId().toString()) + doDelete("/api/asset/"+asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -349,7 +345,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsTitle2) { - doDelete("/api/asset/" + asset.getId().getId().toString()) + doDelete("/api/asset/"+asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -365,10 +361,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i = 0; i < 143; i++) { + for (int i=0;i<143;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1 + suffix; + String name = title1+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -377,10 +373,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i = 0; i < 75; i++) { + for (int i=0;i<75;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2 + suffix; + String name = title2+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -421,7 +417,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsType2, loadedAssetsType2); for (Asset asset : loadedAssetsType1) { - doDelete("/api/asset/" + asset.getId().getId().toString()) + doDelete("/api/asset/"+asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -432,7 +428,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsType2) { - doDelete("/api/asset/" + asset.getId().getId().toString()) + doDelete("/api/asset/"+asset.getId().getId().toString()) .andExpect(status().isOk()); } @@ -451,9 +447,9 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i = 0; i < 128; i++) { + for (int i=0;i<128;i++) { Asset asset = new Asset(); - asset.setName("Asset" + i); + asset.setName("Asset"+i); asset.setType("default"); asset = doPost("/api/asset", asset, Asset.class); assets.add(doPost("/api/customer/" + customerId.getId().toString() @@ -487,10 +483,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i = 0; i < 125; i++) { + for (int i=0;i<125;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1 + suffix; + String name = title1+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -500,10 +496,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i = 0; i < 143; i++) { + for (int i=0;i<143;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2 + suffix; + String name = title2+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -578,10 +574,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i = 0; i < 125; i++) { + for (int i=0;i<125;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1 + suffix; + String name = title1+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -592,10 +588,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i = 0; i < 143; i++) { + for (int i=0;i<143;i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2 + suffix; + String name = title2+suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index 34c39bdccb..4c7667a7c2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -17,7 +17,6 @@ package org.thingsboard.server.controller; import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.type.TypeReference; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.eclipse.paho.client.mqttv3.MqttConnectOptions; @@ -25,7 +24,6 @@ import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.id.CustomerId; @@ -44,7 +42,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; @@ -54,7 +51,6 @@ import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; -@Slf4j public abstract class BaseEntityViewControllerTest extends AbstractControllerTest { private IdComparator idComparator; @@ -421,22 +417,12 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); + Thread.sleep(3000); + MqttMessage message = new MqttMessage(); message.setPayload(strKvs.getBytes()); client.publish("v1/devices/me/telemetry", message); Thread.sleep(1000); - client.disconnect(); - } - - private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException { - long start = System.currentTimeMillis(); - while (!client.isConnected()) { - Thread.sleep(100); - if (start + ms < System.currentTimeMillis()) { - throw new RuntimeException("Client is not connected!"); - } - } } private Set getTelemetryKeys(String type, String id) throws Exception { @@ -463,13 +449,13 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); + Thread.sleep(3000); MqttMessage message = new MqttMessage(); message.setPayload((stringKV).getBytes()); client.publish("v1/devices/me/attributes", message); Thread.sleep(1000); - client.disconnect(); + return new HashSet<>(doGetAsync("/api/plugins/telemetry/DEVICE/" + viewDeviceId + "/keys/attributes", List.class)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index 347eaee7eb..8dc0acff57 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -15,12 +15,10 @@ */ package org.thingsboard.server.controller; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -35,9 +33,4 @@ public class ControllerSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); - - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java index a9bb113a60..c43637e05f 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java @@ -16,13 +16,11 @@ package org.thingsboard.server.mqtt; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; import org.thingsboard.server.dao.CustomSqlUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -43,9 +41,4 @@ public class MqttNoSqlTestSuite { Arrays.asList( new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false)), "cassandra-test.yaml", 30000l); - - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java index 70de3b27ca..2863589ba1 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java @@ -15,12 +15,10 @@ */ package org.thingsboard.server.mqtt; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -34,9 +32,4 @@ public class MqttSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); - - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 9ddb2c81d9..5ba4dfcbae 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -37,7 +37,6 @@ import org.thingsboard.server.service.security.AccessValidator; import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -56,8 +55,6 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC private User tenantAdmin; private Long asyncContextTimeoutToUseRpcPlugin; - private static final AtomicInteger atomicInteger = new AtomicInteger(2); - @Before public void beforeTest() throws Exception { @@ -73,7 +70,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); - tenantAdmin.setEmail("tenant" + atomicInteger.getAndIncrement() + "@thingsboard.org"); + tenantAdmin.setEmail("tenant2@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); @@ -112,8 +109,6 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", MqttQoS.AT_MOST_ONCE.value()); - Thread.sleep(2000); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -133,7 +128,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"24\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -142,7 +137,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttOneWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"25\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + nonExistentDeviceId, setGpioRequest, String.class, @@ -170,9 +165,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", 1); client.setCallback(new TestMqttCallback(client, new CountDownLatch(1))); - Thread.sleep(2000); - - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -190,7 +183,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"27\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -199,7 +192,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttTwoWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"28\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + nonExistentDeviceId, setGpioRequest, String.class, diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index 47e9537ef9..f1580c65c7 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -79,8 +79,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr String deviceId = savedDevice.getId().getId().toString(); - Thread.sleep(2000); - List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); + Thread.sleep(1000); + List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); Set actualKeySet = new HashSet<>(actualKeys); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4"); @@ -88,7 +88,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals(expectedKeySet, actualKeySet); - String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); + String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); Map>> values = doGetAsync(getTelemetryValuesUrl, Map.class); assertEquals("value1", values.get("key1").get(0).get("value")); @@ -97,25 +97,20 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals("4", values.get("key4").get(0).get("value")); } - -// @Test - Unstable + @Test public void testMqttQoSLevel() throws Exception { String clientId = MqttAsyncClient.generateClientId(); MqttAsyncClient client = new MqttAsyncClient(MQTT_URL, clientId); MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); + client.connect(options).waitForCompletion(3000); CountDownLatch latch = new CountDownLatch(1); TestMqttCallback callback = new TestMqttCallback(client, latch); client.setCallback(callback); - client.connect(options).waitForCompletion(5000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); - String payload = "{\"key\":\"uniqueValue\"}"; -// TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. -// MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) -// MqttClient <- SUB_ACK <- Transport - Thread.sleep(5000); - doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); + String payload = "{\"key\":\"value\"}"; + String result = doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); latch.await(10, TimeUnit.SECONDS); assertEquals(payload, callback.getPayload()); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); @@ -125,8 +120,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr private final MqttAsyncClient client; private final CountDownLatch latch; - private volatile Integer qoS; - private volatile String payload; + private Integer qoS; + private String payload; String getPayload() { return payload; @@ -143,7 +138,6 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr @Override public void connectionLost(Throwable throwable) { - log.error("Client connection lost", throwable); } @Override diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java index 83ac5f7703..5f930821f7 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java @@ -15,12 +15,10 @@ */ package org.thingsboard.server.rules; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -35,9 +33,4 @@ public class RuleEngineSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); - - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } } diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 97e6e59969..cfac1ea66b 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -15,21 +15,19 @@ */ package org.thingsboard.server.rules.flow; -import akka.actor.ActorRef; +import com.datastax.driver.core.utils.UUIDs; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.Event; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.User; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; @@ -39,14 +37,17 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.rule.RuleChainService; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -61,7 +62,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule protected User tenantAdmin; @Autowired - protected ActorSystemContext actorSystem; + protected ActorService actorService; @Autowired protected AttributesService attributesService; @@ -148,12 +149,15 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); - TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); - TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); - QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - actorSystem.tell(qMsg, ActorRef.noSender()); - Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + "CUSTOM", + device.getId(), + new TbMsgMetaData(), + "{}", null, null, 0L); + actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + + Thread.sleep(3000); PageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); @@ -260,13 +264,15 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); - TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); - TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); - QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - actorSystem.tell(qMsg, ActorRef.noSender()); - - Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + "CUSTOM", + device.getId(), + new TbMsgMetaData(), + "{}", null, null, 0L); + actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + + Thread.sleep(3000); PageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 088f9390ff..1d35168e78 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -15,17 +15,15 @@ */ package org.thingsboard.server.rules.lifecycle; -import akka.actor.ActorRef; import com.datastax.driver.core.utils.UUIDs; +import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; -import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -40,13 +38,13 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -63,7 +61,7 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac protected User tenantAdmin; @Autowired - protected ActorSystemContext actorSystem; + protected ActorService actorService; @Autowired protected AttributesService attributesService; @@ -138,13 +136,16 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac Thread.sleep(1000); - TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); - TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); - QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - actorSystem.tell(qMsg, ActorRef.noSender()); - Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); - + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + "CUSTOM", + device.getId(), + new TbMsgMetaData(), + "{}", + null, null, 0L); + actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + + Thread.sleep(3000); PageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java new file mode 100644 index 0000000000..d717169048 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.routing; + +import com.datastax.driver.core.utils.UUIDs; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.UUIDConverter; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.cluster.ServerType; +import org.thingsboard.server.service.cluster.discovery.DiscoveryService; +import org.thingsboard.server.service.cluster.discovery.ServerInstance; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class ConsistentClusterRoutingServiceTest { + + private ConsistentClusterRoutingService clusterRoutingService; + + private DiscoveryService discoveryService; + + private String hashFunctionName = "murmur3_128"; + private Integer virtualNodesSize = 1024*4; + private ServerAddress currentServer = new ServerAddress(" 100.96.1.0", 9001, ServerType.CORE); + + + @Before + public void setup() throws Exception { + discoveryService = mock(DiscoveryService.class); + clusterRoutingService = new ConsistentClusterRoutingService(); + ReflectionTestUtils.setField(clusterRoutingService, "discoveryService", discoveryService); + ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); + ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); + when(discoveryService.getCurrentServer()).thenReturn(new ServerInstance(currentServer)); + List otherServers = new ArrayList<>(); + for (int i = 1; i < 30; i++) { + otherServers.add(new ServerInstance(new ServerAddress(" 100.96." + i*2 + "." + i, 9001, ServerType.CORE))); + } + when(discoveryService.getOtherServers()).thenReturn(otherServers); + clusterRoutingService.init(); + } + + @Test + public void testDispersionOnMillionDevices() { + List devices = new ArrayList<>(); + for (int i = 0; i < 1000000; i++) { + devices.add(new DeviceId(UUIDs.timeBased())); + } + + testDevicesDispersion(devices); + } + + private void testDevicesDispersion(List devices) { + long start = System.currentTimeMillis(); + Map map = new HashMap<>(); + for (DeviceId deviceId : devices) { + ServerAddress address = clusterRoutingService.resolveById(deviceId).orElse(currentServer); + map.put(address, map.getOrDefault(address, 0) + 1); + } + + List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); + long end = System.currentTimeMillis(); + System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + (data.get(data.size() - 1).getValue() - data.get(0).getValue())); + + for (Map.Entry entry : data) { +// System.out.println(entry.getKey().getHost() + ": " + entry.getValue()); + } + + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java deleted file mode 100644 index 3f7619ed1e..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import com.datastax.driver.core.utils.UUIDs; -import lombok.extern.slf4j.Slf4j; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.runners.MockitoJUnitRunner; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.test.util.ReflectionTestUtils; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.queue.discovery.ConsistentHashPartitionService; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.mockito.Mockito.mock; - -@Slf4j -@RunWith(MockitoJUnitRunner.class) -public class ConsistentHashParitionServiceTest { - - public static final int ITERATIONS = 1000000; - private ConsistentHashPartitionService clusterRoutingService; - - private TbServiceInfoProvider discoveryService; - private TenantRoutingInfoService routingInfoService; - private ApplicationEventPublisher applicationEventPublisher; - - private String hashFunctionName = "murmur3_128"; - private Integer virtualNodesSize = 16; - - - @Before - public void setup() throws Exception { - discoveryService = mock(TbServiceInfoProvider.class); - applicationEventPublisher = mock(ApplicationEventPublisher.class); - routingInfoService = mock(TenantRoutingInfoService.class); - clusterRoutingService = new ConsistentHashPartitionService(discoveryService, routingInfoService, applicationEventPublisher); - ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); - ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); - ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); - ReflectionTestUtils.setField(clusterRoutingService, "ruleEnginePartitions", 100); - - ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); - ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); - TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() - .setServiceId("100.96.1.1") - .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) - .build(); -// when(discoveryService.getServiceInfo()).thenReturn(currentServer); - List otherServers = new ArrayList<>(); - for (int i = 1; i < 30; i++) { - otherServers.add(TransportProtos.ServiceInfo.newBuilder() - .setServiceId("100.96." + i * 2 + "." + i) - .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) - .build()); - } - clusterRoutingService.init(); - clusterRoutingService.recalculatePartitions(currentServer, otherServers); - } - - @Test - public void testDispersionOnMillionDevices() { - List devices = new ArrayList<>(); - for (int i = 0; i < ITERATIONS; i++) { - devices.add(new DeviceId(UUIDs.timeBased())); - } - testDevicesDispersion(devices); - } - - private void testDevicesDispersion(List devices) { - long start = System.currentTimeMillis(); - Map map = new HashMap<>(); - for (DeviceId deviceId : devices) { - TopicPartitionInfo address = clusterRoutingService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, deviceId); - Integer partition = address.getPartition().get(); - map.put(partition, map.getOrDefault(partition, 0) + 1); - } - - List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); - long end = System.currentTimeMillis(); - double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff / ITERATIONS) * 100.0) + "%)"); - - for (Map.Entry entry : data) { - System.out.println(entry.getKey() + ": " + entry.getValue()); - } - - } - -} diff --git a/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java deleted file mode 100644 index 43cd62ea97..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.runners.MockitoJUnitRunner; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; - -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@Slf4j -@RunWith(MockitoJUnitRunner.class) -public class TbMsgPackProcessingContextTest { - - @Test - public void testHighConcurrencyCase() throws InterruptedException { - TbRuleEngineSubmitStrategy strategyMock = mock(TbRuleEngineSubmitStrategy.class); - int msgCount = 1000; - int parallelCount = 5; - ExecutorService executorService = Executors.newFixedThreadPool(parallelCount); - try { - ConcurrentMap> messages = new ConcurrentHashMap<>(); - for (int i = 0; i < msgCount; i++) { - messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); - } - when(strategyMock.getPendingMap()).thenReturn(messages); - TbMsgPackProcessingContext context = new TbMsgPackProcessingContext(strategyMock); - for (UUID uuid : messages.keySet()) { - for (int i = 0; i < parallelCount; i++) { - executorService.submit(() -> context.onSuccess(uuid)); - } - } - Assert.assertTrue(context.await(10, TimeUnit.SECONDS)); - Mockito.verify(strategyMock, Mockito.times(msgCount)).onSuccess(Mockito.any(UUID.class)); - } finally { - executorService.shutdownNow(); - } - } -} diff --git a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java index 62881bd5af..82b3df9736 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java @@ -24,7 +24,6 @@ import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -63,7 +62,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("70", actual.getMetaData().getValue("temp")); @@ -79,7 +78,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("94", actual.getMetaData().getValue("newAttr")); @@ -95,7 +94,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}"; - TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); TbMsg actual = scriptEngine.executeUpdate(msg); @@ -113,7 +112,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); assertFalse(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -127,7 +126,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); assertTrue(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -148,7 +147,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one"), actual); scriptEngine.destroy(); @@ -170,7 +169,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one", "three"), actual); scriptEngine.destroy(); diff --git a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java index 8a89de5dff..1065fbf6d3 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java +++ b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java @@ -49,9 +49,4 @@ public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } - - @Override - protected long getMaxBlacklistDuration() { - return 100000; - } } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java deleted file mode 100644 index c4182db3ee..0000000000 --- a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.system; - -import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.runner.RunWith; -import org.thingsboard.server.dao.CustomCassandraCQLUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; - -import java.util.Arrays; - -/** - * @author Andrew Shvayka - */ -@RunWith(ClasspathSuite.class) -@ClasspathSuite.ClassnameFilters({"org.thingsboard.server.system.*NoSqlTest"}) -public class SystemNoSqlTestSuite { - - @ClassRule - public static CustomCassandraCQLUnit cassandraUnit = - new CustomCassandraCQLUnit( - Arrays.asList( - new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false), - new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), - new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), - "cassandra-test.yaml", 30000l); - - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } -} diff --git a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java index 6060a2cc2b..b12d513ce0 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java @@ -15,12 +15,10 @@ */ package org.thingsboard.server.system; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -37,9 +35,4 @@ public class SystemSqlTestSuite { "sql/drop-all-tables.sql", "sql-test.properties"); - @BeforeClass - public static void cleanupInMemStorage(){ - InMemoryStorage.getInstance().cleanup(); - } - } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 298800a2f0..bf5a494afb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -18,7 +18,7 @@ package org.thingsboard.server.dao.alarm; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java index a41dd2ee21..96a36e1f65 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.HSQLDialect") public @interface HsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java index bcf1222988..20954b8ac2 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.entities.type}'=='cassandra'") public @interface NoSqlAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java index 7b10e0cf03..ebd0ef7d28 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "cassandra") public @interface NoSqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java index a888926bd0..ef73ec8f90 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.PostgreSQLDialect") public @interface PsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java index a215a16b29..f2a8800032 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale') " + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") public @interface PsqlTsAnyDao { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java index 76db7920a6..c3d607f8fe 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.dao.util; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -@Retention(RetentionPolicy.RUNTIME) public @interface SqlDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java index 9be3321988..9a43c530a2 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale'") public @interface SqlTsAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java index 0b665c5695..abba0e985b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "sql") public @interface SqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java index 0541a47879..20745d790c 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java @@ -17,10 +17,6 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "timescale") public @interface TimescaleDBTsDao { } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java index 766d405e25..b1f6a967b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java @@ -29,8 +29,6 @@ public class Tenant extends ContactBased implements HasTenantId { private String title; private String region; - private boolean isolatedTbCore; - private boolean isolatedTbRuleEngine; public Tenant() { super(); @@ -74,22 +72,6 @@ public class Tenant extends ContactBased implements HasTenantId { this.region = region; } - public boolean isIsolatedTbCore() { - return isolatedTbCore; - } - - public void setIsolatedTbCore(boolean isolatedTbCore) { - this.isolatedTbCore = isolatedTbCore; - } - - public boolean isIsolatedTbRuleEngine() { - return isolatedTbRuleEngine; - } - - public void setIsolatedTbRuleEngine(boolean isolatedTbRuleEngine) { - this.isolatedTbRuleEngine = isolatedTbRuleEngine; - } - @Override public String getSearchText() { return getTitle(); @@ -102,10 +84,6 @@ public class Tenant extends ContactBased implements HasTenantId { builder.append(title); builder.append(", region="); builder.append(region); - builder.append(", isolatedTbCore="); - builder.append(isolatedTbCore); - builder.append(", isolatedTbRuleEngine="); - builder.append(isolatedTbRuleEngine); builder.append(", additionalInfo="); builder.append(getAdditionalInfo()); builder.append(", country="); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java index 9c23a60c28..a43d95975e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java @@ -23,7 +23,6 @@ import lombok.Data; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java rename to common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java index 691d89cafd..532842ae75 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java @@ -13,12 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.id; +package org.thingsboard.server.common.data.alarm; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.UUIDBased; import java.util.UUID; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 11db1da1a0..c367355cb0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.id; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.alarm.AlarmId; import java.util.UUID; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java index 2993d81154..5639f98d01 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java @@ -32,10 +32,6 @@ public class BaseAttributeKvEntry implements AttributeKvEntry { this.lastUpdateTs = lastUpdateTs; } - public BaseAttributeKvEntry(long lastUpdateTs, KvEntry kv) { - this(kv, lastUpdateTs); - } - @Override public long getLastUpdateTs() { return lastUpdateTs; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 0345355aa3..04b0cae56b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -15,9 +15,6 @@ */ package org.thingsboard.server.common.msg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; - /** * Created by ashvayka on 15.03.18. */ @@ -27,12 +24,17 @@ public enum MsgType { /** * ADDED/UPDATED/DELETED events for server nodes. * - * See {@link PartitionChangeMsg} + * See {@link org.thingsboard.server.common.msg.cluster.ClusterEventMsg} */ - PARTITION_CHANGE_MSG, + CLUSTER_EVENT_MSG, APP_INIT_MSG, + /** + * All messages, could be send to cluster + */ + SEND_TO_CLUSTER_MSG, + /** * ADDED/UPDATED/DELETED events for main entities. * @@ -41,11 +43,11 @@ public enum MsgType { COMPONENT_LIFE_CYCLE_MSG, /** - * Misc messages consumed from the Queue and forwarded to Rule Engine Actor. + * Misc messages from the REST API/SERVICE layer to the new rule engine. * - * See {@link QueueToRuleEngineMsg} + * See {@link org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg} */ - QUEUE_TO_RULE_ENGINE_MSG, + SERVICE_TO_RULE_ENGINE_MSG, /** * Message that is sent by RuleChainActor to RuleActor with command to process TbMsg. @@ -89,9 +91,12 @@ public enum MsgType { DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG, + DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG, + /** * Message that is sent from the Device Actor to Rule Engine. Requires acknowledgement */ + DEVICE_ACTOR_TO_RULE_ENGINE_MSG, SESSION_TIMEOUT_MSG, diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 3edf4e5061..4e7090f934 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -15,27 +15,24 @@ */ package org.thingsboard.server.common.msg; -import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import lombok.Builder; +import lombok.AllArgsConstructor; import lombok.Data; -import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; +import java.nio.ByteBuffer; import java.util.UUID; /** * Created by ashvayka on 13.01.18. */ @Data -@Builder -@Slf4j +@AllArgsConstructor public final class TbMsg implements Serializable { private final UUID id; @@ -44,61 +41,30 @@ public final class TbMsg implements Serializable { private final TbMsgMetaData metaData; private final TbMsgDataType dataType; private final String data; + private final TbMsgTransactionData transactionData; + + //The following fields are not persisted to DB, because they can always be recovered from the context; private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; - //This field is not serialized because we use queues and there is no need to do it - private final TbMsgCallback callback; - - public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); - } - - public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); - } - - public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { - return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); - } - - public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); - } - - public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { - return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); - } - - public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), - data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); - } - - public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(UUID.randomUUID(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), - tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); - } + private final long clusterPartition; - private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, String data, + RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { this.id = id; this.type = type; this.originator = originator; this.metaData = metaData; - this.dataType = dataType; this.data = data; + this.dataType = TbMsgDataType.JSON; + this.transactionData = new TbMsgTransactionData(id, originator); this.ruleChainId = ruleChainId; this.ruleNodeId = ruleNodeId; - if (callback != null) { - this.callback = callback; - } else { - log.warn("[{}] Created message with empty callback: {}", originator, type); - this.callback = TbMsgCallback.EMPTY; - } + this.clusterPartition = clusterPartition; } - public static ByteString toByteString(TbMsg msg) { - return ByteString.copyFrom(toByteArray(msg)); + public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, + RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { + this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, clusterPartition); } public static byte[] toByteArray(TbMsg msg) { @@ -123,37 +89,52 @@ public final class TbMsg implements Serializable { builder.setMetaData(MsgProtos.TbMsgMetaDataProto.newBuilder().putAllData(msg.getMetaData().getData()).build()); } + TbMsgTransactionData transactionData = msg.getTransactionData(); + if (transactionData != null) { + MsgProtos.TbMsgTransactionDataProto.Builder transactionBuilder = MsgProtos.TbMsgTransactionDataProto.newBuilder(); + transactionBuilder.setId(transactionData.getTransactionId().toString()); + transactionBuilder.setEntityType(transactionData.getOriginatorId().getEntityType().name()); + transactionBuilder.setEntityIdMSB(transactionData.getOriginatorId().getId().getMostSignificantBits()); + transactionBuilder.setEntityIdLSB(transactionData.getOriginatorId().getId().getLeastSignificantBits()); + builder.setTransactionData(transactionBuilder.build()); + } builder.setDataType(msg.getDataType().ordinal()); builder.setData(msg.getData()); return builder.build().toByteArray(); + } - public static TbMsg fromBytes(byte[] data, TbMsgCallback callback) { + public static ByteBuffer toBytes(TbMsg msg) { + return ByteBuffer.wrap(toByteArray(msg)); + } + + public static TbMsg fromBytes(byte[] data) { + return fromBytes(ByteBuffer.wrap(data)); + } + + public static TbMsg fromBytes(ByteBuffer buffer) { try { - MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data); + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(buffer.array()); TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap()); + EntityId transactionEntityId = EntityIdFactory.getByTypeAndUuid(proto.getTransactionData().getEntityType(), + new UUID(proto.getTransactionData().getEntityIdMSB(), proto.getTransactionData().getEntityIdLSB())); + TbMsgTransactionData transactionData = new TbMsgTransactionData(UUID.fromString(proto.getTransactionData().getId()), transactionEntityId); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - RuleChainId ruleChainId = null; + RuleChainId ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); RuleNodeId ruleNodeId = null; - if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) { - ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); - } if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) { ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); } TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); + return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), transactionData, ruleChainId, ruleNodeId, proto.getClusterPartition()); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } } - public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { - return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); + public TbMsg copy(UUID newId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { + return new TbMsg(newId, type, originator, metaData.copy(), dataType, data, transactionData, ruleChainId, ruleNodeId, clusterPartition); } - public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); - } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java new file mode 100644 index 0000000000..5d57514275 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +import java.io.Serializable; +import java.util.UUID; + +@Data +public final class TbMsgTransactionData implements Serializable { + + private final UUID transactionId; + private final EntityId originatorId; + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java similarity index 71% rename from common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java index 0e8d66aa63..56f928a1c8 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java @@ -13,28 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.queue; +package org.thingsboard.server.common.msg.cluster; import lombok.Data; -import lombok.Getter; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; -import java.util.Set; - /** * @author Andrew Shvayka */ @Data -public final class PartitionChangeMsg implements TbActorMsg { +public final class ClusterEventMsg implements TbActorMsg { - @Getter - private final ServiceQueueKey serviceQueueKey; - @Getter - private final Set partitions; + private final ServerAddress serverAddress; + private final boolean added; @Override public MsgType getMsgType() { - return MsgType.PARTITION_CHANGE_MSG; + return MsgType.CLUSTER_EVENT_MSG; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java similarity index 55% rename from application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java index b258c6db1b..3326c33aa3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java @@ -13,23 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.processing; +package org.thingsboard.server.common.msg.cluster; +import lombok.Data; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.TbActorMsg; -import java.util.UUID; +@Data +public class SendToClusterMsg implements TbActorMsg { -public class SequentialByTenantIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { + private TbActorMsg msg; + private EntityId entityId; - public SequentialByTenantIdTbRuleEngineSubmitStrategy(String queueName) { - super(queueName); + public SendToClusterMsg(EntityId entityId, TbActorMsg msg) { + this.entityId = entityId; + this.msg = msg; } - @Override - protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { - return new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); + @Override + public MsgType getMsgType() { + return MsgType.SEND_TO_CLUSTER_MSG; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java similarity index 50% rename from application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java index 5a5c172ee6..80674f9bce 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java @@ -13,32 +13,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue; +package org.thingsboard.server.common.msg.cluster; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TbCallback; +import lombok.Data; +import lombok.EqualsAndHashCode; -import java.util.UUID; +import java.io.Serializable; -@Slf4j -public class TbPackCallback implements TbCallback { - private final TbPackProcessingContext ctx; - private final UUID id; +/** + * @author Andrew Shvayka + */ +@Data +@EqualsAndHashCode +public class ServerAddress implements Comparable, Serializable { - public TbPackCallback(UUID id, TbPackProcessingContext ctx) { - this.id = id; - this.ctx = ctx; - } + private final String host; + private final int port; + private final ServerType serverType; @Override - public void onSuccess() { - log.trace("[{}] ON SUCCESS", id); - ctx.onSuccess(id); + public int compareTo(ServerAddress o) { + int result = this.host.compareTo(o.host); + if (result == 0) { + result = this.port - o.port; + } + return result; } @Override - public void onFailure(Throwable t) { - log.trace("[{}] ON FAILURE", id, t); - ctx.onFailure(id, t); + public String toString() { + return '[' + host + ':' + port + ']'; } } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java similarity index 82% rename from application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java index d7a4715460..4b00284c34 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java @@ -13,8 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.subscription; +package org.thingsboard.server.common.msg.cluster; -public enum TbSubscriptionType { - TIMESERIES, ATTRIBUTES +/** + * Created by ashvayka on 23.09.18. + */ +public enum ServerType { + CORE } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java b/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java similarity index 76% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java index 2c5da0d3ea..d06046fd71 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java @@ -13,14 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.service; +package org.thingsboard.server.common.msg.core; import lombok.Data; -import java.util.UUID; - +/** + * @author Andrew Shvayka + */ @Data -public class RpcRequestMetadata { - private final UUID sessionId; +public class ToServerRpcResponseMsg { + private final int requestId; + private final String data; + + public boolean isSuccess() { + return true; + } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index d05948206c..a52c237f87 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -33,7 +33,7 @@ import java.util.Optional; * @author Andrew Shvayka */ @ToString -public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { +public class ComponentLifecycleMsg implements TbActorMsg, TenantAwareMsg, ToAllNodesMsg { @Getter private final TenantId tenantId; @Getter diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java deleted file mode 100644 index dd720251bb..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RuleEngineException extends Exception { - protected static final ObjectMapper mapper = new ObjectMapper(); - - public RuleEngineException(String message) { - super(message != null ? message : "Unknown"); - } - - public String toJsonString() { - try { - return mapper.writeValueAsString(mapper.createObjectNode().put("message", getMessage())); - } catch (JsonProcessingException e) { - log.warn("Failed to serialize exception ", e); - throw new RuntimeException(e); - } - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java deleted file mode 100644 index 3f437dd1d7..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -import com.fasterxml.jackson.core.JsonProcessingException; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.RuleNodeId; -import org.thingsboard.server.common.data.rule.RuleNode; - -@Slf4j -public class RuleNodeException extends RuleEngineException { - @Getter - private final String ruleChainName; - @Getter - private final String ruleNodeName; - @Getter - private final RuleChainId ruleChainId; - @Getter - private final RuleNodeId ruleNodeId; - - public RuleNodeException(String message, String ruleChainName, RuleNode ruleNode) { - super(message); - this.ruleChainName = ruleChainName; - this.ruleNodeName = ruleNode.getName(); - this.ruleChainId = ruleNode.getRuleChainId(); - this.ruleNodeId = ruleNode.getId(); - } - - public String toJsonString() { - try { - return mapper.writeValueAsString(mapper.createObjectNode() - .put("ruleNodeId", ruleNodeId.toString()) - .put("ruleChainId", ruleChainId.toString()) - .put("ruleNodeName", ruleNodeName) - .put("ruleChainName", ruleChainName) - .put("message", getMessage())); - } catch (JsonProcessingException e) { - log.warn("Failed to serialize exception ", e); - throw new RuntimeException(e); - } - } - -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java deleted file mode 100644 index be08f054b6..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -import lombok.ToString; - -import java.util.Objects; - -@ToString -public class ServiceQueue { - - public static final String MAIN = "Main"; - - private final ServiceType type; - private final String queue; - - public ServiceQueue(ServiceType type) { - this.type = type; - this.queue = MAIN; - } - - public ServiceQueue(ServiceType type, String queue) { - this.type = type; - this.queue = queue; - } - - public ServiceType getType() { - return type; - } - - public String getQueue() { - return queue; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ServiceQueue that = (ServiceQueue) o; - return type == that.type && - queue.equals(that.queue); - } - - @Override - public int hashCode() { - return Objects.hash(type, queue); - } - -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java deleted file mode 100644 index 0ca776126d..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -public enum ServiceType { - - TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; - - public static ServiceType of(String serviceType) { - return ServiceType.valueOf(serviceType.replace("-", "_").toUpperCase()); - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java deleted file mode 100644 index 9e8d5ae6b8..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -public interface TbMsgCallback { - - TbMsgCallback EMPTY = new TbMsgCallback() { - - @Override - public void onSuccess() { - - } - - @Override - public void onFailure(RuleEngineException e) { - - } - }; - - void onSuccess(); - - void onFailure(RuleEngineException e); - -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java deleted file mode 100644 index bf0fffe404..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.queue; - -import lombok.Builder; -import lombok.Getter; -import lombok.ToString; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.Objects; -import java.util.Optional; - -@ToString -public class TopicPartitionInfo { - - private final String topic; - private final TenantId tenantId; - private final Integer partition; - @Getter - private final String fullTopicName; - @Getter - private final boolean myPartition; - - @Builder - public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { - this.topic = topic; - this.tenantId = tenantId; - this.partition = partition; - this.myPartition = myPartition; - String tmp = topic; - if (tenantId != null) { - tmp += "." + tenantId.getId().toString(); - } - if (partition != null) { - tmp += "." + partition; - } - this.fullTopicName = tmp; - } - - public String getTopic() { - return topic; - } - - public Optional getTenantId() { - return Optional.ofNullable(tenantId); - } - - public Optional getPartition() { - return Optional.ofNullable(partition); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TopicPartitionInfo that = (TopicPartitionInfo) o; - return topic.equals(that.topic) && - Objects.equals(tenantId, that.tenantId) && - Objects.equals(partition, that.partition) && - fullTopicName.equals(that.fullTopicName); - } - - @Override - public int hashCode() { - return Objects.hash(fullTopicName); - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java similarity index 73% rename from common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java index fca1c51995..a717af85a4 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.queue; +package org.thingsboard.server.common.msg.system; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; @@ -22,25 +22,18 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import java.io.Serializable; -import java.util.Set; /** * Created by ashvayka on 15.03.18. */ @Data -public final class QueueToRuleEngineMsg implements TbActorMsg { +public final class ServiceToRuleEngineMsg implements TbActorMsg, Serializable { private final TenantId tenantId; private final TbMsg tbMsg; - private final Set relationTypes; - private final String failureMessage; @Override public MsgType getMsgType() { - return MsgType.QUEUE_TO_RULE_ENGINE_MSG; - } - - public boolean isTellNext() { - return relationTypes != null && !relationTypes.isEmpty(); + return MsgType.SERVICE_TO_RULE_ENGINE_MSG; } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java new file mode 100644 index 0000000000..5d47425e43 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.timeout; + +import org.thingsboard.server.common.msg.MsgType; + +/** + * @author Andrew Shvayka + */ +public final class DeviceActorClientSideRpcTimeoutMsg extends TimeoutMsg { + + public DeviceActorClientSideRpcTimeoutMsg(Integer id, long timeout) { + super(id, timeout); + } + + @Override + public MsgType getMsgType() { + return MsgType.DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG; + } +} diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 737d5f6f57..03dc17b5c9 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -23,6 +23,13 @@ message TbMsgMetaDataProto { map data = 1; } +message TbMsgTransactionDataProto { + string id = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; +} + message TbMsgProto { string id = 1; string type = 2; @@ -39,7 +46,7 @@ message TbMsgProto { TbMsgMetaDataProto metaData = 11; - //Transaction Data (12) was removed in 2.5 + TbMsgTransactionDataProto transactionData = 12; int32 dataType = 13; string data = 14; diff --git a/common/queue/pom.xml b/common/queue/pom.xml index b4a223cac0..24881a0e3f 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -40,10 +40,6 @@ org.thingsboard.common data - - org.thingsboard.common - util - org.thingsboard.common message @@ -52,22 +48,6 @@ org.apache.kafka kafka-clients - - com.amazonaws - aws-java-sdk-sqs - - - com.google.cloud - google-cloud-pubsub - - - com.microsoft.azure - azure-servicebus - - - com.rabbitmq - amqp-client - org.springframework spring-context-support @@ -104,14 +84,6 @@ ch.qos.logback logback-classic - - com.google.protobuf - protobuf-java - - - org.apache.curator - curator-recipes - junit junit @@ -124,13 +96,4 @@ - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - - diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java similarity index 70% rename from common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java index ec3b0b537a..0c68c8dd5d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java @@ -13,17 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.common; +package org.thingsboard.server.kafka; + +import lombok.extern.slf4j.Slf4j; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.UUID; -public class AbstractTbQueueTemplate { - protected static final String REQUEST_ID_HEADER = "requestId"; - protected static final String RESPONSE_TOPIC_HEADER = "responseTopic"; - protected static final String REQUEST_TIME = "requestTime"; - +/** + * Created by ashvayka on 25.09.18. + */ +@Slf4j +public abstract class AbstractTbKafkaTemplate { protected byte[] uuidToBytes(UUID uuid) { ByteBuffer buf = ByteBuffer.allocate(16); buf.putLong(uuid.getMostSignificantBits()); @@ -45,14 +47,4 @@ public class AbstractTbQueueTemplate { protected String bytesToString(byte[] data) { return new String(data, StandardCharsets.UTF_8); } - - protected static byte[] longToBytes(long x) { - ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); - longBuffer.putLong(0, x); - return longBuffer.array(); - } - - protected static long bytesToLong(byte[] bytes) { - return ByteBuffer.wrap(bytes).getLong(); - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java similarity index 98% rename from common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java index aa87df031a..a01411e48f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.common; +package org.thingsboard.server.kafka; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java new file mode 100644 index 0000000000..0fe86e030b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.kafka; + +import org.apache.kafka.clients.admin.*; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.KafkaFuture; + +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Created by ashvayka on 24.09.18. + */ +public class TBKafkaAdmin { + + AdminClient client; + + public TBKafkaAdmin(TbKafkaSettings settings) { + client = AdminClient.create(settings.toProps()); + } + + public void waitForTopic(String topic, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException { + synchronized (this) { + long timeoutExpiredMs = System.currentTimeMillis() + timeoutUnit.toMillis(timeout); + while (!topicExists(topic)) { + long waitMs = timeoutExpiredMs - System.currentTimeMillis(); + if (waitMs <= 0) { + throw new TimeoutException("Timeout occurred while waiting for topic [" + topic + "] to be available!"); + } else { + wait(1000); + } + } + } + } + + public CreateTopicsResult createTopic(NewTopic topic){ + return client.createTopics(Collections.singletonList(topic)); + } + + private boolean topicExists(String topic) throws InterruptedException { + KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); + try { + topicDescriptionFuture.get(); + return true; + } catch (ExecutionException e) { + return false; + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java new file mode 100644 index 0000000000..73f8cc16c7 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java @@ -0,0 +1,88 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.kafka; + +import lombok.Builder; +import lombok.Getter; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; +import java.util.UUID; + +/** + * Created by ashvayka on 24.09.18. + */ +public class TBKafkaConsumerTemplate { + + private final KafkaConsumer consumer; + private final TbKafkaDecoder decoder; + + @Builder.Default + private TbKafkaRequestIdExtractor requestIdExtractor = ((response) -> null); + + @Getter + private final String topic; + + @Builder + private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, + TbKafkaRequestIdExtractor requestIdExtractor, + String clientId, String groupId, String topic, + boolean autoCommit, int autoCommitIntervalMs, + int maxPollRecords) { + Properties props = settings.toProps(); + props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + if (groupId != null) { + props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + } + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit); + props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + if (maxPollRecords > 0) { + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); + } + this.consumer = new KafkaConsumer<>(props); + this.decoder = decoder; + this.requestIdExtractor = requestIdExtractor; + this.topic = topic; + } + + public void subscribe() { + consumer.subscribe(Collections.singletonList(topic)); + } + + public void unsubscribe() { + consumer.unsubscribe(); + } + + public ConsumerRecords poll(Duration duration) { + return consumer.poll(duration); + } + + public T decode(ConsumerRecord record) throws IOException { + return decoder.decode(record.value()); + } + + public UUID extractRequestId(T value) { + return requestIdExtractor.extractRequestId(value); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java new file mode 100644 index 0000000000..8722c1d64e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -0,0 +1,131 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.kafka; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.errors.TopicExistsException; +import org.apache.kafka.common.header.Header; +import org.springframework.util.StringUtils; + +import java.util.List; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +/** + * Created by ashvayka on 24.09.18. + */ +@Slf4j +public class TBKafkaProducerTemplate { + + private final KafkaProducer producer; + private final TbKafkaEncoder encoder; + + private final TbKafkaPartitioner partitioner; + private ConcurrentMap> partitionInfoMap; + @Getter + private final String defaultTopic; + + @Getter + private final TbKafkaSettings settings; + + @Builder + private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaEncoder encoder, + TbKafkaPartitioner partitioner, String defaultTopic, String clientId) { + Properties props = settings.toProps(); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + if (!StringUtils.isEmpty(clientId)) { + props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + } + this.settings = settings; + this.producer = new KafkaProducer<>(props); + this.encoder = encoder; + this.partitioner = partitioner; + this.defaultTopic = defaultTopic; + } + + public void init() { + this.partitionInfoMap = new ConcurrentHashMap<>(); + if (!StringUtils.isEmpty(defaultTopic)) { + try { + TBKafkaAdmin admin = new TBKafkaAdmin(this.settings); + admin.waitForTopic(defaultTopic, 30, TimeUnit.SECONDS); + log.info("[{}] Topic exists.", defaultTopic); + } catch (Exception e) { + log.info("[{}] Failed to wait for topic: {}", defaultTopic, e.getMessage(), e); + throw new RuntimeException(e); + } + //Maybe this should not be cached, but we don't plan to change size of partitions + this.partitionInfoMap.putIfAbsent(defaultTopic, producer.partitionsFor(defaultTopic)); + } + } + + public Future send(String key, T value, Callback callback) { + return send(key, value, null, callback); + } + + public Future send(String key, T value, Iterable
headers, Callback callback) { + return send(key, value, null, headers, callback); + } + + public Future send(String key, T value, Long timestamp, Iterable
headers, Callback callback) { + if (!StringUtils.isEmpty(this.defaultTopic)) { + return send(this.defaultTopic, key, value, timestamp, headers, callback); + } else { + throw new RuntimeException("Failed to send message! Default topic is not specified!"); + } + } + + public Future send(String topic, String key, T value, Iterable
headers, Callback callback) { + return send(topic, key, value, null, headers, callback); + } + + public Future send(String topic, String key, T value, Callback callback) { + return send(topic, key, value, null, null, callback); + } + + public Future send(String topic, String key, T value, Long timestamp, Iterable
headers, Callback callback) { + byte[] data = encoder.encode(value); + ProducerRecord record; + Integer partition = getPartition(topic, key, value, data); + record = new ProducerRecord<>(topic, partition, timestamp, key, data, headers); + return producer.send(record, callback); + } + + private Integer getPartition(String topic, String key, T value, byte[] data) { + if (partitioner == null) { + return null; + } else { + return partitioner.partition(topic, key, value, data, partitionInfoMap.computeIfAbsent(topic, producer::partitionsFor)); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java similarity index 83% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java index 6e3ea1e4a9..ab196d5863 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java @@ -13,9 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.kafka; - -import org.thingsboard.server.queue.TbQueueMsg; +package org.thingsboard.server.kafka; import java.io.IOException; @@ -24,6 +22,6 @@ import java.io.IOException; */ public interface TbKafkaDecoder { - T decode(TbQueueMsg msg) throws IOException; + T decode(byte[] data) throws IOException; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java index b3c4dec8ee..22f65c58a0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.kafka; +package org.thingsboard.server.kafka; /** * Created by ashvayka on 25.09.18. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java index 609cbd2107..5e5aa0591a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.kafka; import com.google.common.util.concurrent.ListenableFuture; /** * Created by ashvayka on 05.10.18. */ -public interface TbQueueHandler { +public interface TbKafkaHandler { ListenableFuture handle(Request request); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java similarity index 63% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java index 1b2f984093..8ec5cd1d97 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java @@ -13,25 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.kafka; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.apache.kafka.clients.producer.Partitioner; +import org.apache.kafka.common.PartitionInfo; import java.util.List; -import java.util.Set; -public interface TbQueueConsumer { - - String getTopic(); - - void subscribe(); - - void subscribe(Set partitions); - - void unsubscribe(); - - List poll(long durationInMillis); +/** + * Created by ashvayka on 25.09.18. + */ +public interface TbKafkaPartitioner extends Partitioner { - void commit(); + int partition(String topic, String key, T value, byte[] encodedValue, List partitions); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java similarity index 74% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java index b6de7db50d..cd52948da8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java @@ -13,9 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.kafka; +package org.thingsboard.server.kafka; import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; /** * Created by ashvayka on 25.09.18. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java similarity index 81% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java index b0e4990305..6e433e71a9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java @@ -13,15 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.kafka; import java.util.UUID; -public interface TbQueueMsg { +public interface TbKafkaRequestIdExtractor { - UUID getKey(); + UUID extractRequestId(T value); - TbQueueMsgHeaders getHeaders(); - - byte[] getData(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java similarity index 54% rename from common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java index f02ae63441..2a7bca119f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java @@ -13,22 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.common; +package org.thingsboard.server.kafka; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.TopicExistsException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -37,14 +41,15 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeoutException; +/** + * Created by ashvayka on 25.09.18. + */ @Slf4j -public class DefaultTbQueueRequestTemplate extends AbstractTbQueueTemplate - implements TbQueueRequestTemplate { +public class TbKafkaRequestTemplate extends AbstractTbKafkaTemplate { - private final TbQueueAdmin queueAdmin; - private final TbQueueProducer requestTemplate; - private final TbQueueConsumer responseTemplate; - private final ConcurrentMap> pendingRequests; + private final TBKafkaProducerTemplate requestTemplate; + private final TBKafkaConsumerTemplate responseTemplate; + private final ConcurrentMap> pendingRequests; private final boolean internalExecutor; private final ExecutorService executor; private final long maxRequestTimeout; @@ -55,14 +60,12 @@ public class DefaultTbQueueRequestTemplate requestTemplate, - TbQueueConsumer responseTemplate, - long maxRequestTimeout, - long maxPendingRequests, - long pollInterval, - ExecutorService executor) { - this.queueAdmin = queueAdmin; + public TbKafkaRequestTemplate(TBKafkaProducerTemplate requestTemplate, + TBKafkaConsumerTemplate responseTemplate, + long maxRequestTimeout, + long maxPendingRequests, + long pollInterval, + ExecutorService executor) { this.requestTemplate = requestTemplate; this.responseTemplate = responseTemplate; this.pendingRequests = new ConcurrentHashMap<>(); @@ -78,9 +81,20 @@ public class DefaultTbQueueRequestTemplate responses = responseTemplate.poll(pollInterval); - if (responses.size() > 0) { - log.trace("Polling responses completed, consumer records count [{}]", responses.size()); - } else { - continue; + ConsumerRecords responses = responseTemplate.poll(Duration.ofMillis(pollInterval)); + if (responses.count() > 0) { + log.trace("Polling responses completed, consumer records count [{}]", responses.count()); } responses.forEach(response -> { - byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); - UUID requestId; + log.trace("Received response to Kafka Template request: {}", response); + Header requestIdHeader = response.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); + Response decodedResponse = null; + UUID requestId = null; if (requestIdHeader == null) { + try { + decodedResponse = responseTemplate.decode(response); + requestId = responseTemplate.extractRequestId(decodedResponse); + } catch (IOException e) { + log.error("Failed to decode response", e); + } + } else { + requestId = bytesToUuid(requestIdHeader.value()); + } + if (requestId == null) { log.error("[{}] Missing requestId in header and body", response); } else { - requestId = bytesToUuid(requestIdHeader); - log.trace("[{}] Response received: {}", requestId, response); + log.trace("[{}] Response received", requestId); ResponseMetaData expectedResponse = pendingRequests.remove(requestId); if (expectedResponse == null) { log.trace("[{}] Invalid or stale request", requestId); } else { - expectedResponse.future.set(response); + try { + if (decodedResponse == null) { + decodedResponse = responseTemplate.decode(response); + } + expectedResponse.future.set(decodedResponse); + } catch (IOException e) { + expectedResponse.future.setException(e); + } } } }); - responseTemplate.commit(); tickTs = System.currentTimeMillis(); tickSize = pendingRequests.size(); if (nextCleanupMs < tickTs) { @@ -126,6 +155,10 @@ public class DefaultTbQueueRequestTemplate send(Request request) { + public ListenableFuture post(String key, Request request) { if (tickSize > maxPendingRequests) { return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); } UUID requestId = UUID.randomUUID(); - request.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); - request.getHeaders().put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic())); - request.getHeaders().put(REQUEST_TIME, longToBytes(System.currentTimeMillis())); + List
headers = new ArrayList<>(2); + headers.add(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))); + headers.add(new RecordHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic()))); SettableFuture future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime); - requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() { - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - log.trace("[{}] Request sent: {}", requestId, metadata); - } - - @Override - public void onFailure(Throwable t) { - pendingRequests.remove(requestId); - future.setException(t); + log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime); + requestTemplate.send(key, request, headers, (metadata, exception) -> { + if (exception != null) { + log.trace("[{}] Failed to post the request", requestId, exception); + } else { + log.trace("[{}] Posted the request: {}", requestId, metadata); } }); return future; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java new file mode 100644 index 0000000000..864182a552 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.kafka; + +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; + +import java.time.Duration; +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by ashvayka on 25.09.18. + */ +@Slf4j +public class TbKafkaResponseTemplate extends AbstractTbKafkaTemplate { + + private final TBKafkaConsumerTemplate requestTemplate; + private final TBKafkaProducerTemplate responseTemplate; + private final TbKafkaHandler handler; + private final ConcurrentMap pendingRequests; + private final ExecutorService loopExecutor; + private final ScheduledExecutorService timeoutExecutor; + private final ExecutorService callbackExecutor; + private final int maxPendingRequests; + private final long requestTimeout; + + private final long pollInterval; + private volatile boolean stopped = false; + private final AtomicInteger pendingRequestCount = new AtomicInteger(); + + @Builder + public TbKafkaResponseTemplate(TBKafkaConsumerTemplate requestTemplate, + TBKafkaProducerTemplate responseTemplate, + TbKafkaHandler handler, + long pollInterval, + long requestTimeout, + int maxPendingRequests, + ExecutorService executor) { + this.requestTemplate = requestTemplate; + this.responseTemplate = responseTemplate; + this.handler = handler; + this.pendingRequests = new ConcurrentHashMap<>(); + this.maxPendingRequests = maxPendingRequests; + this.pollInterval = pollInterval; + this.requestTimeout = requestTimeout; + this.callbackExecutor = executor; + this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); + this.loopExecutor = Executors.newSingleThreadExecutor(); + } + + public void init() { + this.responseTemplate.init(); + requestTemplate.subscribe(); + loopExecutor.submit(() -> { + while (!stopped) { + try { + while (pendingRequestCount.get() >= maxPendingRequests) { + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e) { + log.trace("Failed to wait until the server has capacity to handle new requests", e); + } + } + ConsumerRecords requests = requestTemplate.poll(Duration.ofMillis(pollInterval)); + requests.forEach(request -> { + Header requestIdHeader = request.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header", request); + return; + } + UUID requestId = bytesToUuid(requestIdHeader.value()); + if (requestId == null) { + log.error("[{}] Missing requestId in header and body", request); + return; + } + Header responseTopicHeader = request.headers().lastHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER); + if (responseTopicHeader == null) { + log.error("[{}] Missing response topic in header", request); + return; + } + String responseTopic = bytesToString(responseTopicHeader.value()); + try { + pendingRequestCount.getAndIncrement(); + Request decodedRequest = requestTemplate.decode(request); + AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(decodedRequest), + response -> { + pendingRequestCount.decrementAndGet(); + reply(requestId, responseTopic, response); + }, + e -> { + pendingRequestCount.decrementAndGet(); + if (e.getCause() != null && e.getCause() instanceof TimeoutException) { + log.warn("[{}] Timedout to process the request: {}", requestId, request, e); + } else { + log.trace("[{}] Failed to process the request: {}", requestId, request, e); + } + }, + requestTimeout, + timeoutExecutor, + callbackExecutor); + } catch (Throwable e) { + pendingRequestCount.decrementAndGet(); + log.warn("[{}] Failed to process the request: {}", requestId, request, e); + } + }); + } catch (InterruptException ie) { + if (!stopped) { + log.warn("Fetching data from kafka was interrupted.", ie); + } + } catch (Throwable e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + public void stop() { + stopped = true; + if (timeoutExecutor != null) { + timeoutExecutor.shutdownNow(); + } + if (loopExecutor != null) { + loopExecutor.shutdownNow(); + } + } + + private void reply(UUID requestId, String topic, Response response) { + responseTemplate.send(topic, requestId.toString(), response, Collections.singletonList(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))), null); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java similarity index 79% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java index 66121cb215..6446643a18 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.kafka; +package org.thingsboard.server.kafka; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import java.util.List; @@ -29,32 +28,31 @@ import java.util.Properties; * Created by ashvayka on 25.09.18. */ @Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +@ConditionalOnProperty(prefix = "kafka", value = "enabled", havingValue = "true", matchIfMissing = false) @Component public class TbKafkaSettings { - @Value("${queue.kafka.bootstrap.servers}") + static final String REQUEST_ID_HEADER = "requestId"; + static final String RESPONSE_TOPIC_HEADER = "responseTopic"; + + @Value("${kafka.bootstrap.servers}") private String servers; - @Value("${queue.kafka.acks}") + @Value("${kafka.acks}") private String acks; - @Value("${queue.kafka.retries}") + @Value("${kafka.retries}") private int retries; - @Value("${queue.kafka.batch.size}") + @Value("${kafka.batch.size}") private int batchSize; - @Value("${queue.kafka.linger.ms}") + @Value("${kafka.linger.ms}") private long lingerMs; - @Value("${queue.kafka.buffer.memory}") + @Value("${kafka.buffer.memory}") private long bufferMemory; - @Value("${queue.kafka.replication_factor}") - @Getter - private short replicationFactor; - @Value("${kafka.other:#{null}}") private List other; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java similarity index 50% rename from common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java rename to common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java index 922a2b1062..343c5e6dd8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java @@ -13,30 +13,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.sqs; +package org.thingsboard.server.kafka; -import lombok.Data; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import javax.annotation.PostConstruct; +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * Created by ashvayka on 12.10.18. + */ @Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") @Component -@Data -public class TbAwsSqsSettings { - - @Value("${queue.aws_sqs.access_key_id}") - private String accessKeyId; - - @Value("${queue.aws_sqs.secret_access_key}") - private String secretAccessKey; +public class TbNodeIdProvider { - @Value("${queue.aws_sqs.region}") - private String region; + @Getter + @Value("${cluster.node_id:#{null}}") + private String nodeId; - @Value("${queue.aws_sqs.threads_per_topic}") - private int threadsPerTopic; + @PostConstruct + public void init() { + if (StringUtils.isEmpty(nodeId)) { + try { + nodeId = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + nodeId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); + } + } + log.info("Current NodeId: {}", nodeId); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java deleted file mode 100644 index f21ad80d10..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -public interface TbQueueCallback { - - void onSuccess(TbQueueMsgMetadata metadata); - - void onFailure(Throwable t); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java deleted file mode 100644 index 4cb38f6685..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -import com.google.protobuf.InvalidProtocolBufferException; -import org.thingsboard.server.queue.TbQueueMsg; - -public interface TbQueueMsgDecoder { - - T decode(TbQueueMsg msg) throws InvalidProtocolBufferException; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java deleted file mode 100644 index f205ed676f..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -import java.util.Map; - -public interface TbQueueMsgHeaders { - - byte[] put(String key, byte[] value); - - byte[] get(String key); - - Map getData(); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java deleted file mode 100644 index 0c24fec438..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -public interface TbQueueMsgMetadata { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java deleted file mode 100644 index ef308291a9..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; - -public interface TbQueueProducer { - - void init(); - - String getDefaultTopic(); - - void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback); - - void stop(); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java deleted file mode 100644 index 20530360d6..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -import com.google.common.util.concurrent.ListenableFuture; - -public interface TbQueueRequestTemplate { - - void init(); - - ListenableFuture send(Request request); - - void stop(); - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java deleted file mode 100644 index c823b3df88..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue; - -public interface TbQueueResponseTemplate { - - void init(TbQueueHandler handler); - - void stop(); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java deleted file mode 100644 index 336514b7b2..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import com.microsoft.azure.servicebus.management.ManagementClient; -import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; -import com.microsoft.azure.servicebus.primitives.ServiceBusException; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.io.IOException; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -@Slf4j -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") -public class TbServiceBusAdmin implements TbQueueAdmin { - - private final Set queues = ConcurrentHashMap.newKeySet(); - - private final ManagementClient client; - - public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings) { - ConnectionStringBuilder builder = new ConnectionStringBuilder( - serviceBusSettings.getNamespaceName(), - "queues", - serviceBusSettings.getSasKeyName(), - serviceBusSettings.getSasKey()); - - client = new ManagementClient(builder); - - try { - client.getQueues().forEach(queueDescription -> queues.add(queueDescription.getPath())); - } catch (ServiceBusException | InterruptedException e) { - log.error("Failed to get queues.", e); - throw new RuntimeException("Failed to get queues.", e); - } - } - - @Override - public void createTopicIfNotExists(String topic) { - if (queues.contains(topic)) { - return; - } - - try { - client.createQueue(topic); - queues.add(topic); - } catch (ServiceBusException | InterruptedException e) { - log.error("Failed to create queue: [{}]", topic, e); - } - } - - public void destroy() { - try { - client.close(); - } catch (IOException e) { - log.error("Failed to close ManagementClient."); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java deleted file mode 100644 index cca599d59a..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.microsoft.azure.servicebus.TransactionContext; -import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; -import com.microsoft.azure.servicebus.primitives.CoreMessageReceiver; -import com.microsoft.azure.servicebus.primitives.MessageWithDeliveryTag; -import com.microsoft.azure.servicebus.primitives.MessagingEntityType; -import com.microsoft.azure.servicebus.primitives.MessagingFactory; -import com.microsoft.azure.servicebus.primitives.SettleModePair; -import lombok.extern.slf4j.Slf4j; -import org.apache.qpid.proton.amqp.messaging.Data; -import org.apache.qpid.proton.amqp.transport.ReceiverSettleMode; -import org.apache.qpid.proton.amqp.transport.SenderSettleMode; -import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.time.Duration; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -@Slf4j -public class TbServiceBusConsumerTemplate implements TbQueueConsumer { - private final TbQueueAdmin admin; - private final String topic; - private final TbQueueMsgDecoder decoder; - private final TbServiceBusSettings serviceBusSettings; - - private final Gson gson = new Gson(); - - private Set receivers; - private volatile Set partitions; - private volatile boolean subscribed; - private volatile boolean stopped = false; - private Map> pendingMessages = new ConcurrentHashMap<>(); - private volatile int messagesPerQueue; - - public TbServiceBusConsumerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String topic, TbQueueMsgDecoder decoder) { - this.admin = admin; - this.decoder = decoder; - this.topic = topic; - this.serviceBusSettings = serviceBusSettings; - } - - @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public void unsubscribe() { - stopped = true; - receivers.forEach(CoreMessageReceiver::closeAsync); - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - createReceivers(); - messagesPerQueue = receivers.size() / partitions.size(); - subscribed = true; - } - - List>> messageFutures = - receivers.stream() - .map(receiver -> receiver - .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) - .whenComplete((messages, err) -> { - if (!CollectionUtils.isEmpty(messages)) { - pendingMessages.put(receiver, messages); - } else if (err != null) { - log.error("Failed to receive messages.", err); - } - })) - .collect(Collectors.toList()); - try { - return fromList(messageFutures) - .get() - .stream() - .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) - .map(message -> { - try { - return decode(message); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to parse message.", e); - throw new RuntimeException("Failed to parse message.", e); - } - }).collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", topic); - } else { - log.error("Failed to receive messages", e); - } - } - } - return Collections.emptyList(); - } - - private void createReceivers() { - List> receiverFutures = partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .map(queue -> { - MessagingFactory factory; - try { - factory = MessagingFactory.createFromConnectionStringBuilder(createConnection(queue)); - } catch (InterruptedException | ExecutionException e) { - log.error("Failed to create factory for the queue [{}]", queue); - throw new RuntimeException("Failed to create the factory", e); - } - - return CoreMessageReceiver.create(factory, queue, queue, 0, - new SettleModePair(SenderSettleMode.UNSETTLED, ReceiverSettleMode.SECOND), - MessagingEntityType.QUEUE); - }).collect(Collectors.toList()); - - try { - receivers = new HashSet<>(fromList(receiverFutures).get()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", topic); - } else { - log.error("Failed to create receivers", e); - } - } - } - - private ConnectionStringBuilder createConnection(String queue) { - admin.createTopicIfNotExists(queue); - return new ConnectionStringBuilder( - serviceBusSettings.getNamespaceName(), - queue, - serviceBusSettings.getSasKeyName(), - serviceBusSettings.getSasKey()); - } - - private CompletableFuture> fromList(List> futures) { - CompletableFuture>[] arrayFuture = new CompletableFuture[futures.size()]; - futures.toArray(arrayFuture); - - return CompletableFuture - .allOf(arrayFuture) - .thenApply(v -> futures - .stream() - .map(CompletableFuture::join) - .collect(Collectors.toList())); - } - - @Override - public void commit() { - pendingMessages.forEach((receiver, msgs) -> - msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); - pendingMessages.clear(); - } - - private T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(new String(((Data) data.getMessage().getBody()).getValue().getArray()), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java deleted file mode 100644 index 5d9a931378..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import com.google.gson.Gson; -import com.microsoft.azure.servicebus.IMessage; -import com.microsoft.azure.servicebus.Message; -import com.microsoft.azure.servicebus.QueueClient; -import com.microsoft.azure.servicebus.ReceiveMode; -import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; -import com.microsoft.azure.servicebus.primitives.ServiceBusException; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -@Slf4j -public class TbServiceBusProducerTemplate implements TbQueueProducer { - private final String defaultTopic; - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final TbServiceBusSettings serviceBusSettings; - private final Map clients = new HashMap<>(); - private ExecutorService executorService; - - public TbServiceBusProducerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String defaultTopic) { - this.admin = admin; - this.defaultTopic = defaultTopic; - this.serviceBusSettings = serviceBusSettings; - executorService = Executors.newSingleThreadExecutor(); - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - IMessage message = new Message(gson.toJson(new DefaultTbQueueMsg(msg))); - CompletableFuture future = getClient(tpi.getFullTopicName()).sendAsync(message); - future.whenCompleteAsync((success, err) -> { - if (err != null) { - callback.onFailure(err); - } else { - callback.onSuccess(null); - } - }, executorService); - } - - @Override - public void stop() { - clients.forEach((t, client) -> { - try { - client.close(); - } catch (ServiceBusException e) { - log.error("Failed to close QueueClient.", e); - } - }); - - if (executorService != null) { - executorService.shutdownNow(); - } - } - - private QueueClient getClient(String topic) { - return clients.computeIfAbsent(topic, k -> { - admin.createTopicIfNotExists(topic); - ConnectionStringBuilder builder = - new ConnectionStringBuilder( - serviceBusSettings.getNamespaceName(), - topic, - serviceBusSettings.getSasKeyName(), - serviceBusSettings.getSasKey()); - try { - return new QueueClient(builder, ReceiveMode.PEEKLOCK); - } catch (InterruptedException | ServiceBusException e) { - log.error("Failed to create new client for the Queue: [{}]", topic, e); - throw new RuntimeException("Failed to create new client for the Queue", e); - } - }); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java deleted file mode 100644 index d872dcec0b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -@Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") -@Component -@Data -public class TbServiceBusSettings { - @Value("${queue.service_bus.namespace_name}") - private String namespaceName; - @Value("${queue.service_bus.sas_key_name}") - private String sasKeyName; - @Value("${queue.service_bus.sas_key}") - private String sasKey; - @Value("${queue.service_bus.max_messages}") - private int maxMessages; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java deleted file mode 100644 index 7584e8c2d0..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import lombok.Data; -import org.thingsboard.server.queue.TbQueueMsg; - -import java.util.UUID; - -@Data -public class DefaultTbQueueMsg implements TbQueueMsg { - private final UUID key; - private final byte[] data; - private final DefaultTbQueueMsgHeaders headers; - - public DefaultTbQueueMsg(TbQueueMsg msg) { - this.key = msg.getKey(); - this.data = msg.getData(); - DefaultTbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); - msg.getHeaders().getData().forEach(headers::put); - this.headers = headers; - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java deleted file mode 100644 index 36c3dd0999..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import org.thingsboard.server.queue.TbQueueMsgHeaders; - -import java.util.HashMap; -import java.util.Map; - -public class DefaultTbQueueMsgHeaders implements TbQueueMsgHeaders { - - protected final Map data = new HashMap<>(); - - @Override - public byte[] put(String key, byte[] value) { - return data.put(key, value); - } - - @Override - public byte[] get(String key) { - return data.get(key); - } - - @Override - public Map getData() { - return data; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java deleted file mode 100644 index 96891ee7d8..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import lombok.Builder; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueHandler; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueResponseTemplate; - -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; - -@Slf4j -public class DefaultTbQueueResponseTemplate extends AbstractTbQueueTemplate - implements TbQueueResponseTemplate { - - private final TbQueueConsumer requestTemplate; - private final TbQueueProducer responseTemplate; - private final ConcurrentMap pendingRequests; - private final ExecutorService loopExecutor; - private final ScheduledExecutorService timeoutExecutor; - private final ExecutorService callbackExecutor; - private final int maxPendingRequests; - private final long requestTimeout; - - private final long pollInterval; - private volatile boolean stopped = false; - private final AtomicInteger pendingRequestCount = new AtomicInteger(); - - @Builder - public DefaultTbQueueResponseTemplate(TbQueueConsumer requestTemplate, - TbQueueProducer responseTemplate, - TbQueueHandler handler, - long pollInterval, - long requestTimeout, - int maxPendingRequests, - ExecutorService executor) { - this.requestTemplate = requestTemplate; - this.responseTemplate = responseTemplate; - this.pendingRequests = new ConcurrentHashMap<>(); - this.maxPendingRequests = maxPendingRequests; - this.pollInterval = pollInterval; - this.requestTimeout = requestTimeout; - this.callbackExecutor = executor; - this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); - this.loopExecutor = Executors.newSingleThreadExecutor(); - } - - @Override - public void init(TbQueueHandler handler) { - this.responseTemplate.init(); - requestTemplate.subscribe(); - loopExecutor.submit(() -> { - while (!stopped) { - try { - while (pendingRequestCount.get() >= maxPendingRequests) { - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e) { - log.trace("Failed to wait until the server has capacity to handle new requests", e); - } - } - List requests = requestTemplate.poll(pollInterval); - - if (requests.isEmpty()) { - continue; - } - - requests.forEach(request -> { - long currentTime = System.currentTimeMillis(); - long requestTime = bytesToLong(request.getHeaders().get(REQUEST_TIME)); - if (requestTime + requestTimeout >= currentTime) { - byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); - if (requestIdHeader == null) { - log.error("[{}] Missing requestId in header", request); - return; - } - byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); - if (responseTopicHeader == null) { - log.error("[{}] Missing response topic in header", request); - return; - } - UUID requestId = bytesToUuid(requestIdHeader); - String responseTopic = bytesToString(responseTopicHeader); - try { - pendingRequestCount.getAndIncrement(); - AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), - response -> { - pendingRequestCount.decrementAndGet(); - response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); - responseTemplate.send(TopicPartitionInfo.builder().topic(responseTopic).build(), response, null); - }, - e -> { - pendingRequestCount.decrementAndGet(); - if (e.getCause() != null && e.getCause() instanceof TimeoutException) { - log.warn("[{}] Timeout to process the request: {}", requestId, request, e); - } else { - log.trace("[{}] Failed to process the request: {}", requestId, request, e); - } - }, - requestTimeout, - timeoutExecutor, - callbackExecutor); - } catch (Throwable e) { - pendingRequestCount.decrementAndGet(); - log.warn("[{}] Failed to process the request: {}", requestId, request, e); - } - } - }); - requestTemplate.commit(); - } catch (Throwable e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - public void stop() { - stopped = true; - if (requestTemplate != null) { - requestTemplate.unsubscribe(); - } - if (responseTemplate != null) { - responseTemplate.stop(); - } - if (timeoutExecutor != null) { - timeoutExecutor.shutdownNow(); - } - if (loopExecutor != null) { - loopExecutor.shutdownNow(); - } - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java deleted file mode 100644 index f860d8dcc9..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; - -import java.util.concurrent.atomic.AtomicInteger; - -public class MultipleTbQueueTbMsgCallbackWrapper implements TbQueueCallback { - - private final AtomicInteger tbQueueCallbackCount; - private final TbMsgCallback tbMsgCallback; - - public MultipleTbQueueTbMsgCallbackWrapper(int tbQueueCallbackCount, TbMsgCallback tbMsgCallback) { - this.tbQueueCallbackCount = new AtomicInteger(tbQueueCallbackCount); - this.tbMsgCallback = tbMsgCallback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (tbQueueCallbackCount.decrementAndGet() <= 0) { - tbMsgCallback.onSuccess(); - } - } - - @Override - public void onFailure(Throwable t) { - tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java deleted file mode 100644 index 07417c4a9e..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import org.thingsboard.server.queue.TbQueueMsgHeaders; - -import java.nio.charset.StandardCharsets; -import java.util.UUID; - -public class TbProtoJsQueueMsg extends TbProtoQueueMsg { - - public TbProtoJsQueueMsg(UUID key, T value) { - super(key, value); - } - - public TbProtoJsQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { - super(key, value, headers); - } - - @Override - public byte[] getData() { - try { - return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java deleted file mode 100644 index 2eb76950dc..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import lombok.Data; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgHeaders; - -import java.util.UUID; - -@Data -public class TbProtoQueueMsg implements TbQueueMsg { - - private final UUID key; - protected final T value; - private final TbQueueMsgHeaders headers; - - public TbProtoQueueMsg(UUID key, T value) { - this(key, value, new DefaultTbQueueMsgHeaders()); - } - - public TbProtoQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { - this.key = key; - this.value = value; - this.headers = headers; - } - - @Override - public UUID getKey() { - return key; - } - - @Override - public TbQueueMsgHeaders getHeaders() { - return headers; - } - - @Override - public byte[] getData() { - return value.toByteArray(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java deleted file mode 100644 index 8bd9c3516b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.common; - -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; - -public class TbQueueTbMsgCallbackWrapper implements TbQueueCallback { - - private final TbMsgCallback tbMsgCallback; - - public TbQueueTbMsgCallbackWrapper(TbMsgCallback tbMsgCallback) { - this.tbMsgCallback = tbMsgCallback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - tbMsgCallback.onSuccess(); - } - - @Override - public void onFailure(Throwable t) { - tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java deleted file mode 100644 index f0039d0f38..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import lombok.Getter; -import org.springframework.context.ApplicationEvent; -import org.thingsboard.server.common.msg.queue.ServiceQueueKey; - -import java.util.Set; - - -public class ClusterTopologyChangeEvent extends ApplicationEvent { - - @Getter - private final Set serviceQueueKeys; - - public ClusterTopologyChangeEvent(Object source, Set serviceQueueKeys) { - super(source); - this.serviceQueueKeys = serviceQueueKeys; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java deleted file mode 100644 index df25ea3ba9..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ /dev/null @@ -1,339 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import com.google.common.hash.HashCode; -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceQueueKey; -import org.thingsboard.server.common.msg.queue.ServiceQueue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; - -import javax.annotation.PostConstruct; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.stream.Collectors; - -@Service -@Slf4j -public class ConsistentHashPartitionService implements PartitionService { - - @Value("${queue.core.topic}") - private String coreTopic; - @Value("${queue.core.partitions:100}") - private Integer corePartitions; - @Value("${queue.partitions.hash_function_name:murmur3_128}") - private String hashFunctionName; - @Value("${queue.partitions.virtual_nodes_size:16}") - private Integer virtualNodesSize; - - private final ApplicationEventPublisher applicationEventPublisher; - private final TbServiceInfoProvider serviceInfoProvider; - private final TenantRoutingInfoService tenantRoutingInfoService; - private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); - private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); - private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); - - private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); - private ConcurrentMap tpiCache = new ConcurrentHashMap<>(); - - private Map tbCoreNotificationTopics = new HashMap<>(); - private Map tbRuleEngineNotificationTopics = new HashMap<>(); - private List currentOtherServices; - - private HashFunction hashFunction; - - public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, TenantRoutingInfoService tenantRoutingInfoService, ApplicationEventPublisher applicationEventPublisher) { - this.serviceInfoProvider = serviceInfoProvider; - this.tenantRoutingInfoService = tenantRoutingInfoService; - this.applicationEventPublisher = applicationEventPublisher; - } - - @PostConstruct - public void init() { - this.hashFunction = forName(hashFunctionName); - partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions); - partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic); - } - - @Override - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { - return resolve(new ServiceQueue(serviceType), tenantId, entityId); - } - - @Override - public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { - return resolve(new ServiceQueue(serviceType, queueName), tenantId, entityId); - } - - private TopicPartitionInfo resolve(ServiceQueue serviceQueue, TenantId tenantId, EntityId entityId) { - int hash = hashFunction.newHasher() - .putLong(entityId.getId().getMostSignificantBits()) - .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); - Integer partitionSize = partitionSizes.get(serviceQueue); - int partition; - if (partitionSize != null) { - partition = Math.abs(hash % partitionSize); - } else { - //TODO: In 2.6/3.1 this should not happen because all Rule Engine Queues will be in the DB and we always know their partition sizes. - partition = 0; - } - boolean isolatedTenant = isIsolated(serviceQueue, tenantId); - TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceQueue, isolatedTenant ? tenantId : null, partition); - return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceQueue, tenantId, partition)); - } - - @Override - public void recalculatePartitions(ServiceInfo currentService, List otherServices) { - logServiceInfo(currentService); - otherServices.forEach(this::logServiceInfo); - Map> circles = new HashMap<>(); - addNode(circles, currentService); - for (ServiceInfo other : otherServices) { - addNode(circles, other); - } - ConcurrentMap> oldPartitions = myPartitions; - TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService); - myPartitions = new ConcurrentHashMap<>(); - partitionSizes.forEach((type, size) -> { - ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(type, myIsolatedOrSystemTenantId); - for (int i = 0; i < size; i++) { - ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceQueueKey), i); - if (currentService.equals(serviceInfo)) { - ServiceQueueKey serviceQueueKey = new ServiceQueueKey(type, getSystemOrIsolatedTenantId(serviceInfo)); - myPartitions.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(i); - } - } - }); - myPartitions.forEach((serviceQueueKey, partitions) -> { - if (!partitions.equals(oldPartitions.get(serviceQueueKey))) { - log.info("[{}] NEW PARTITIONS: {}", serviceQueueKey, partitions); - Set tpiList = partitions.stream() - .map(partition -> buildTopicPartitionInfo(serviceQueueKey, partition)) - .collect(Collectors.toSet()); - applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, tpiList)); - } - }); - tpiCache.clear(); - - if (currentOtherServices == null) { - currentOtherServices = new ArrayList<>(otherServices); - } else { - Set changes = new HashSet<>(); - Map> currentMap = getServiceKeyListMap(currentOtherServices); - Map> newMap = getServiceKeyListMap(otherServices); - currentOtherServices = otherServices; - currentMap.forEach((key, list) -> { - if (!list.equals(newMap.get(key))) { - changes.add(key); - } - }); - currentMap.keySet().forEach(newMap::remove); - changes.addAll(newMap.keySet()); - if (!changes.isEmpty()) { - applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); - } - } - } - - @Override - public Set getAllServiceIds(ServiceType serviceType) { - Set result = new HashSet<>(); - ServiceInfo current = serviceInfoProvider.getServiceInfo(); - if (current.getServiceTypesList().contains(serviceType.name())) { - result.add(current.getServiceId()); - } - for (ServiceInfo serviceInfo : currentOtherServices) { - if (serviceInfo.getServiceTypesList().contains(serviceType.name())) { - result.add(serviceInfo.getServiceId()); - } - } - return result; - } - - @Override - public TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId) { - switch (serviceType) { - case TB_CORE: - return tbCoreNotificationTopics.computeIfAbsent(serviceId, - id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); - case TB_RULE_ENGINE: - return tbRuleEngineNotificationTopics.computeIfAbsent(serviceId, - id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); - default: - return buildNotificationsTopicPartitionInfo(serviceType, serviceId); - } - } - - private Map> getServiceKeyListMap(List services) { - final Map> currentMap = new HashMap<>(); - services.forEach(serviceInfo -> { - for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { - ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); - if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { - for (TransportProtos.QueueInfo queue : serviceInfo.getRuleEngineQueuesList()) { - ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), getSystemOrIsolatedTenantId(serviceInfo)); - currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); - } - } else { - ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), getSystemOrIsolatedTenantId(serviceInfo)); - currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); - } - } - }); - return currentMap; - } - - private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { - return new TopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); - } - - private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueueKey serviceQueueKey, int partition) { - return buildTopicPartitionInfo(serviceQueueKey.getServiceQueue(), serviceQueueKey.getTenantId(), partition); - } - - private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueue serviceQueue, TenantId tenantId, int partition) { - TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); - tpi.topic(partitionTopics.get(serviceQueue)); - tpi.partition(partition); - ServiceQueueKey myPartitionsSearchKey; - if (isIsolated(serviceQueue, tenantId)) { - tpi.tenantId(tenantId); - myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, tenantId); - } else { - myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, new TenantId(TenantId.NULL_UUID)); - } - List partitions = myPartitions.get(myPartitionsSearchKey); - if (partitions != null) { - tpi.myPartition(partitions.contains(partition)); - } else { - tpi.myPartition(false); - } - return tpi.build(); - } - - private boolean isIsolated(ServiceQueue serviceQueue, TenantId tenantId) { - if (TenantId.SYS_TENANT_ID.equals(tenantId)) { - return false; - } - TenantRoutingInfo routingInfo = tenantRoutingInfoMap.get(tenantId); - if (routingInfo == null) { - synchronized (tenantRoutingInfoMap) { - routingInfo = tenantRoutingInfoMap.get(tenantId); - if (routingInfo == null) { - routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); - tenantRoutingInfoMap.put(tenantId, routingInfo); - } - } - } - if (routingInfo == null) { - throw new RuntimeException("Tenant not found!"); - } - switch (serviceQueue.getType()) { - case TB_CORE: - return routingInfo.isIsolatedTbCore(); - case TB_RULE_ENGINE: - return routingInfo.isIsolatedTbRuleEngine(); - default: - return false; - } - } - - private void logServiceInfo(TransportProtos.ServiceInfo server) { - TenantId tenantId = getSystemOrIsolatedTenantId(server); - if (tenantId.isNullUid()) { - log.info("[{}] Found common server: [{}]", server.getServiceId(), server.getServiceTypesList()); - } else { - log.info("[{}][{}] Found specific server: [{}]", server.getServiceId(), tenantId, server.getServiceTypesList()); - } - } - - private TenantId getSystemOrIsolatedTenantId(TransportProtos.ServiceInfo serviceInfo) { - return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB())); - } - - private void addNode(Map> circles, ServiceInfo instance) { - TenantId tenantId = getSystemOrIsolatedTenantId(instance); - for (String serviceTypeStr : instance.getServiceTypesList()) { - ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); - if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { - for (TransportProtos.QueueInfo queue : instance.getRuleEngineQueuesList()) { - ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), tenantId); - partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getPartitions()); - partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getTopic()); - for (int i = 0; i < virtualNodesSize; i++) { - circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); - } - } - } else { - ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), tenantId); - for (int i = 0; i < virtualNodesSize; i++) { - circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); - } - } - } - } - - private ServiceInfo resolveByPartitionIdx(ConsistentHashCircle circle, Integer partitionIdx) { - if (circle == null || circle.isEmpty()) { - return null; - } - Long hash = hashFunction.newHasher().putInt(partitionIdx).hash().asLong(); - if (!circle.containsKey(hash)) { - ConcurrentNavigableMap tailMap = circle.tailMap(hash); - hash = tailMap.isEmpty() ? - circle.firstKey() : tailMap.firstKey(); - } - return circle.get(hash); - } - - private HashCode hash(ServiceInfo instance, int i) { - return hashFunction.newHasher().putString(instance.getServiceId(), StandardCharsets.UTF_8).putInt(i).hash(); - } - - public static HashFunction forName(String name) { - switch (name) { - case "murmur3_32": - return Hashing.murmur3_32(); - case "murmur3_128": - return Hashing.murmur3_128(); - case "crc32": - return Hashing.crc32(); - case "md5": - return Hashing.md5(); - default: - throw new IllegalArgumentException("Can't find hash function with name " + name); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java deleted file mode 100644 index 1c60a03cdd..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import javax.annotation.PostConstruct; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Collectors; - -@Component -@Slf4j -public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { - - @Getter - @Value("${service.id:#{null}}") - private String serviceId; - - @Getter - @Value("${service.type:monolith}") - private String serviceType; - - @Getter - @Value("${service.tenant_id:}") - private String tenantIdStr; - - @Autowired(required = false) - private TbQueueRuleEngineSettings ruleEngineSettings; - - private List serviceTypes; - private ServiceInfo serviceInfo; - private TenantId isolatedTenant; - - @PostConstruct - public void init() { - if (StringUtils.isEmpty(serviceId)) { - try { - serviceId = InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - serviceId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); - } - } - log.info("Current Service ID: {}", serviceId); - if (serviceType.equalsIgnoreCase("monolith")) { - serviceTypes = Collections.unmodifiableList(Arrays.asList(ServiceType.values())); - } else { - serviceTypes = Collections.singletonList(ServiceType.of(serviceType)); - } - ServiceInfo.Builder builder = ServiceInfo.newBuilder() - .setServiceId(serviceId) - .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())); - UUID tenantId; - if (!StringUtils.isEmpty(tenantIdStr)) { - tenantId = UUID.fromString(tenantIdStr); - isolatedTenant = new TenantId(tenantId); - } else { - tenantId = TenantId.NULL_UUID; - } - builder.setTenantIdMSB(tenantId.getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); - - if (serviceTypes.contains(ServiceType.TB_RULE_ENGINE) && ruleEngineSettings != null) { - for (TbRuleEngineQueueConfiguration queue : ruleEngineSettings.getQueues()) { - TransportProtos.QueueInfo queueInfo = TransportProtos.QueueInfo.newBuilder() - .setName(queue.getName()) - .setTopic(queue.getTopic()) - .setPartitions(queue.getPartitions()).build(); - builder.addRuleEngineQueues(queueInfo); - } - } - - serviceInfo = builder.build(); - } - - @Override - public ServiceInfo getServiceInfo() { - return serviceInfo; - } - - @Override - public boolean isService(ServiceType serviceType) { - return serviceTypes.contains(serviceType); - } - - @Override - public Optional getIsolatedTenant() { - return Optional.ofNullable(isolatedTenant); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java deleted file mode 100644 index 19b8f665a8..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import lombok.Getter; -import org.springframework.context.ApplicationEvent; -import org.thingsboard.server.common.msg.queue.ServiceQueueKey; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; - -import java.util.Set; - - -public class PartitionChangeEvent extends ApplicationEvent { - - @Getter - private final ServiceQueueKey serviceQueueKey; - @Getter - private final Set partitions; - - public PartitionChangeEvent(Object source, ServiceQueueKey serviceQueueKey, Set partitions) { - super(source); - this.serviceQueueKey = serviceQueueKey; - this.partitions = partitions; - } - - public ServiceType getServiceType() { - return serviceQueueKey.getServiceQueue().getType(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java deleted file mode 100644 index e3b3e76b80..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.List; -import java.util.Set; - -/** - * Once application is ready or cluster topology changes, this Service will produce {@link PartitionChangeEvent} - */ -public interface PartitionService { - - TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); - - TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId); - - /** - * Received from the Discovery service when network topology is changed. - * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} - * @param otherServices - all other discovered services {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} - */ - void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); - - /** - * Get all active service ids by service type - * @param serviceType to filter the list of services - * @return list of all active services - */ - Set getAllServiceIds(ServiceType serviceType); - - /** - * Each Service should start a consumer for messages that target individual service instance based on serviceId. - * This topic is likely to have single partition, and is always assigned to the service. - * @param serviceType - * @param serviceId - * @return - */ - TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java deleted file mode 100644 index 07c4c2d9c5..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import lombok.Data; -import org.thingsboard.server.common.data.id.TenantId; - -@Data -public class TenantRoutingInfo { - private final TenantId tenantId; - private final boolean isolatedTbCore; - private final boolean isolatedTbRuleEngine; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java deleted file mode 100644 index 685a36df30..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import org.thingsboard.server.common.data.id.TenantId; - -public interface TenantRoutingInfoService { - - TenantRoutingInfo getRoutingInfo(TenantId tenantId); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java deleted file mode 100644 index 37661e85e4..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.discovery; - -import lombok.AllArgsConstructor; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceQueue; - -import java.util.Objects; - -@AllArgsConstructor -public class TopicPartitionInfoKey { - private ServiceQueue serviceQueue; - private TenantId isolatedTenantId; - private int partition; - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TopicPartitionInfoKey that = (TopicPartitionInfoKey) o; - return partition == that.partition && - serviceQueue.equals(that.serviceQueue) && - Objects.equals(isolatedTenantId, that.isolatedTenantId); - } - - @Override - public int hashCode() { - return Objects.hash(serviceQueue, isolatedTenantId, partition); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java deleted file mode 100644 index 98470eafba..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgHeaders; -import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; - -import java.util.UUID; - -public class KafkaTbQueueMsg implements TbQueueMsg { - private final UUID key; - private final TbQueueMsgHeaders headers; - private final byte[] data; - - public KafkaTbQueueMsg(ConsumerRecord record) { - this.key = UUID.fromString(record.key()); - TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); - record.headers().forEach(header -> { - headers.put(header.key(), header.value()); - }); - this.headers = headers; - this.data = record.value(); - } - - @Override - public UUID getKey() { - return key; - } - - @Override - public TbQueueMsgHeaders getHeaders() { - return headers; - } - - @Override - public byte[] getData() { - return data; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java deleted file mode 100644 index 5a9eaa632c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.thingsboard.server.queue.TbQueueMsgMetadata; - -@Data -@AllArgsConstructor -public class KafkaTbQueueMsgMetadata implements TbQueueMsgMetadata { - private RecordMetadata metadata; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java deleted file mode 100644 index b92b094af1..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.AdminClient; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.common.errors.TopicExistsException; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.util.Collections; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; - -/** - * Created by ashvayka on 24.09.18. - */ -@Slf4j -public class TbKafkaAdmin implements TbQueueAdmin { - - private final AdminClient client; - private final Map topicConfigs; - private final Set topics = ConcurrentHashMap.newKeySet(); - - private final short replicationFactor; - - public TbKafkaAdmin(TbKafkaSettings settings, Map topicConfigs) { - client = AdminClient.create(settings.toProps()); - this.topicConfigs = topicConfigs; - - try { - topics.addAll(client.listTopics().names().get()); - } catch (InterruptedException | ExecutionException e) { - log.error("Failed to get all topics.", e); - } - - replicationFactor = settings.getReplicationFactor(); - } - - @Override - public void createTopicIfNotExists(String topic) { - if (topics.contains(topic)) { - return; - } - try { - NewTopic newTopic = new NewTopic(topic, 1, replicationFactor).configs(topicConfigs); - createTopic(newTopic).values().get(topic).get(); - topics.add(topic); - } catch (ExecutionException ee) { - if (ee.getCause() instanceof TopicExistsException) { - //do nothing - } else { - log.warn("[{}] Failed to create topic", topic, ee); - throw new RuntimeException(ee); - } - } catch (Exception e) { - log.warn("[{}] Failed to create topic", topic, e); - throw new RuntimeException(e); - } - - } - - @Override - public void destroy() { - if (client != null) { - client.close(); - } - } - - public CreateTopicsResult createTopic(NewTopic topic) { - return client.createTopics(Collections.singletonList(topic)); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java deleted file mode 100644 index 49d1d74102..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import lombok.Builder; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; - -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; - -/** - * Created by ashvayka on 24.09.18. - */ -@Slf4j -public class TbKafkaConsumerTemplate implements TbQueueConsumer { - - private final TbQueueAdmin admin; - private final KafkaConsumer consumer; - private final TbKafkaDecoder decoder; - private volatile boolean subscribed; - private volatile Set partitions; - private final Lock consumerLock; - - @Getter - private final String topic; - - @Builder - private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, - String clientId, String groupId, String topic, - boolean autoCommit, int autoCommitIntervalMs, - int maxPollRecords, - TbQueueAdmin admin) { - Properties props = settings.toProps(); - props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - if (groupId != null) { - props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); - } - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit); - props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - if (maxPollRecords > 0) { - props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); - } - this.admin = admin; - this.consumer = new KafkaConsumer<>(props); - this.decoder = decoder; - this.topic = topic; - this.consumerLock = new ReentrantLock(); - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - try { - consumerLock.lock(); - - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); - subscribed = true; - } - - ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); - if (records.count() > 0) { - List result = new ArrayList<>(); - records.forEach(record -> { - try { - result.add(decode(record)); - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - } - }); - return result; - } - } finally { - consumerLock.unlock(); - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - try { - consumerLock.lock(); - consumer.commitAsync(); - } finally { - consumerLock.unlock(); - } - } - - @Override - public void unsubscribe() { - try { - consumerLock.lock(); - if (consumer != null) { - consumer.unsubscribe(); - consumer.close(); - } - } finally { - consumerLock.unlock(); - } - } - - public T decode(ConsumerRecord record) throws IOException { - return decoder.decode(new KafkaTbQueueMsg(record)); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java deleted file mode 100644 index 4f26f51da7..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import lombok.Builder; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.internals.RecordHeader; -import org.springframework.util.StringUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; - -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -/** - * Created by ashvayka on 24.09.18. - */ -@Slf4j -public class TbKafkaProducerTemplate implements TbQueueProducer { - - private final KafkaProducer producer; - - @Getter - private final String defaultTopic; - - @Getter - private final TbKafkaSettings settings; - - private final TbQueueAdmin admin; - - private final Set topics; - - @Builder - private TbKafkaProducerTemplate(TbKafkaSettings settings, String defaultTopic, String clientId, TbQueueAdmin admin) { - Properties props = settings.toProps(); - props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); - props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); - if (!StringUtils.isEmpty(clientId)) { - props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - } - this.settings = settings; - this.producer = new KafkaProducer<>(props); - this.defaultTopic = defaultTopic; - this.admin = admin; - topics = ConcurrentHashMap.newKeySet(); - } - - @Override - public void init() { - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - createTopicIfNotExist(tpi); - String key = msg.getKey().toString(); - byte[] data = msg.getData(); - ProducerRecord record; - Iterable
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); - record = new ProducerRecord<>(tpi.getFullTopicName(), null, key, data, headers); - producer.send(record, (metadata, exception) -> { - if (exception == null) { - if (callback != null) { - callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); - } - } else { - if (callback != null) { - callback.onFailure(exception); - } else { - log.warn("Producer template failure: {}", exception.getMessage(), exception); - } - } - }); - } - - private void createTopicIfNotExist(TopicPartitionInfo tpi) { - if (topics.contains(tpi)) { - return; - } - admin.createTopicIfNotExists(tpi.getFullTopicName()); - topics.add(tpi); - } - - @Override - public void stop() { - if (producer != null) { - producer.close(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java deleted file mode 100644 index 4b68abf929..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka'") -public class TbKafkaTopicConfigs { - @Value("${queue.kafka.topic-properties.core}") - private String coreProperties; - @Value("${queue.kafka.topic-properties.rule-engine}") - private String ruleEngineProperties; - @Value("${queue.kafka.topic-properties.transport-api}") - private String transportApiProperties; - @Value("${queue.kafka.topic-properties.notifications}") - private String notificationsProperties; - @Value("${queue.kafka.topic-properties.js-executor}") - private String jsExecutorProperties; - - @Getter - private Map coreConfigs; - @Getter - private Map ruleEngineConfigs; - @Getter - private Map transportApiConfigs; - @Getter - private Map notificationsConfigs; - @Getter - private Map jsExecutorConfigs; - - @PostConstruct - private void init() { - coreConfigs = getConfigs(coreProperties); - ruleEngineConfigs = getConfigs(ruleEngineProperties); - transportApiConfigs = getConfigs(transportApiProperties); - notificationsConfigs = getConfigs(notificationsProperties); - jsExecutorConfigs = getConfigs(jsExecutorProperties); - } - - private Map getConfigs(String properties) { - Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - configs.put(key, value); - } - return configs; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java deleted file mode 100644 index f51f5b8ae0..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.memory; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.queue.TbQueueMsg; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -@Slf4j -public final class InMemoryStorage { - private static InMemoryStorage instance; - private final ConcurrentHashMap> storage; - - private InMemoryStorage() { - storage = new ConcurrentHashMap<>(); - } - - public static InMemoryStorage getInstance() { - if (instance == null) { - synchronized (InMemoryStorage.class) { - if (instance == null) { - instance = new InMemoryStorage(); - } - } - } - return instance; - } - - public boolean put(String topic, TbQueueMsg msg) { - return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); - } - - public List get(String topic) throws InterruptedException { - if (storage.containsKey(topic)) { - List entities; - T first = (T) storage.get(topic).poll(); - if (first != null) { - entities = new ArrayList<>(); - entities.add(first); - List otherList = new ArrayList<>(); - storage.get(topic).drainTo(otherList, 999); - for (TbQueueMsg other : otherList) { - entities.add((T) other); - } - } else { - entities = Collections.emptyList(); - } - return entities; - } - return Collections.emptyList(); - } - - /** - * Used primarily for testing. - */ - public void cleanup() { - storage.clear(); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java deleted file mode 100644 index a619eda132..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.memory; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; - -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -@Slf4j -public class InMemoryTbQueueConsumer implements TbQueueConsumer { - private final InMemoryStorage storage = InMemoryStorage.getInstance(); - private volatile Set partitions; - private volatile boolean stopped; - private volatile boolean subscribed; - - public InMemoryTbQueueConsumer(String topic) { - this.topic = topic; - stopped = false; - } - - private final String topic; - - @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = true; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = true; - } - - @Override - public void unsubscribe() { - stopped = true; - } - - @Override - public List poll(long durationInMillis) { - if (subscribed) { - List messages = partitions - .stream() - .map(tpi -> { - try { - return storage.get(tpi.getFullTopicName()); - } catch (InterruptedException e) { - if (!stopped) { - log.error("Queue was interrupted.", e); - } - return Collections.emptyList(); - } - }) - .flatMap(List::stream) - .map(msg -> (T) msg).collect(Collectors.toList()); - if (messages.size() > 0) { - return messages; - } - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - if (!stopped) { - log.error("Failed to sleep.", e); - } - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java deleted file mode 100644 index cfcd788a16..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.memory; - -import lombok.Data; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; - -@Data -public class InMemoryTbQueueProducer implements TbQueueProducer { - - private final InMemoryStorage storage = InMemoryStorage.getInstance(); - - private final String defaultTopic; - - public InMemoryTbQueueProducer(String defaultTopic) { - this.defaultTopic = defaultTopic; - } - - @Override - public void init() { - - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - boolean result = storage.put(tpi.getFullTopicName(), msg); - if (result) { - if (callback != null) { - callback.onSuccess(null); - } - } else { - if (callback != null) { - callback.onFailure(new RuntimeException("Failure add msg to InMemoryQueue")); - } - } - } - - @Override - public void stop() { - - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java deleted file mode 100644 index 76ff04c238..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ /dev/null @@ -1,169 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") -public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbAwsSqsSettings sqsSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public AwsSqsMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.sqsSettings = sqsSettings; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); - this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); - this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java deleted file mode 100644 index a7349091bd..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") -public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbAwsSqsSettings sqsSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbAwsSqsQueueAttributes sqsQueueAttributes) { - this.sqsSettings = sqsSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); - this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); - this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java deleted file mode 100644 index 3056eced2c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") -public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbAwsSqsSettings sqsSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - - public AwsSqsTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbAwsSqsSettings sqsSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.sqsSettings = sqsSettings; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); - this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java deleted file mode 100644 index 351cd4058c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbAwsSqsSettings sqsSettings; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public AwsSqsTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes) { - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.sqsSettings = sqsSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbAwsSqsProducerTemplate> producerTemplate = - new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); - - TbAwsSqsConsumerTemplate> consumerTemplate = - new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, - transportApiSettings.getResponsesTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(producerTemplate); - templateBuilder.responseTemplate(consumerTemplate); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java deleted file mode 100644 index fbdbeaab0c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; -import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -@Slf4j -@Component -@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") -public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - - public InMemoryMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(transportNotificationSettings.getNotificationsTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java deleted file mode 100644 index 2855e577ee..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; -import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory { - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - - public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings) { - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - InMemoryTbQueueProducer> producerTemplate = - new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - - InMemoryTbQueueConsumer> consumerTemplate = - new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - - templateBuilder.queueAdmin(new TbQueueAdmin() { - @Override - public void createTopicIfNotExists(String topic) {} - - @Override - public void destroy() {} - }); - - templateBuilder.requestTemplate(producerTemplate); - templateBuilder.responseTemplate(consumerTemplate); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java deleted file mode 100644 index 72c6f7b8b9..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") -public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbKafkaSettings kafkaSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbKafkaTopicConfigs kafkaTopicConfigs) { - this.partitionService = partitionService; - this.kafkaSettings = kafkaSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.jsInvokeSettings = jsInvokeSettings; - - this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); - requestBuilder.admin(notificationAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); - requestBuilder.admin(ruleEngineAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); - requestBuilder.admin(ruleEngineAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - String queueName = configuration.getName(); - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(ruleEngineAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); - consumerBuilder.clientId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(notificationAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(coreSettings.getTopic()); - consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(coreAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - consumerBuilder.clientId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(notificationAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(transportApiSettings.getRequestsTopic()); - consumerBuilder.clientId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(transportApiAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); - requestBuilder.admin(transportApiAdmin); - return requestBuilder.build(); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); - requestBuilder.admin(jsExecutorAdmin); - - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); - responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); - responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); - responseBuilder.decoder(msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - } - ); - responseBuilder.admin(jsExecutorAdmin); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java deleted file mode 100644 index ca10830981..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ /dev/null @@ -1,221 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") -public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { - - private final PartitionService partitionService; - private final TbKafkaSettings kafkaSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public KafkaTbCoreQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbKafkaTopicConfigs kafkaTopicConfigs) { - this.partitionService = partitionService; - this.kafkaSettings = kafkaSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.jsInvokeSettings = jsInvokeSettings; - - this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); - requestBuilder.admin(ruleEngineAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(coreSettings.getTopic()); - consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-core-node-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(coreAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - consumerBuilder.clientId("tb-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-core-notifications-node-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(notificationAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(transportApiSettings.getRequestsTopic()); - consumerBuilder.clientId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(transportApiAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); - requestBuilder.admin(jsExecutorAdmin); - - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); - responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); - responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); - responseBuilder.decoder(msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - } - ); - responseBuilder.admin(jsExecutorAdmin); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java deleted file mode 100644 index bcac399a64..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,193 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") -public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbKafkaSettings kafkaSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - - public KafkaTbRuleEngineQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbKafkaTopicConfigs kafkaTopicConfigs) { - this.partitionService = partitionService; - this.kafkaSettings = kafkaSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.jsInvokeSettings = jsInvokeSettings; - - this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); - requestBuilder.admin(ruleEngineAdmin); - return requestBuilder.build(); - } - - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - String queueName = configuration.getName(); - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(ruleEngineAdmin); - return consumerBuilder.build(); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); - consumerBuilder.clientId("tb-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-rule-engine-notifications-node-" + serviceInfoProvider.getServiceId()); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(notificationAdmin); - return consumerBuilder.build(); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); - requestBuilder.admin(jsExecutorAdmin); - - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); - responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); - responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); - responseBuilder.decoder(msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - } - ); - responseBuilder.admin(jsExecutorAdmin); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java deleted file mode 100644 index 2f7aab4779..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { - - private final TbKafkaSettings kafkaSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public KafkaTbTransportQueueFactory(TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbKafkaTopicConfigs kafkaTopicConfigs) { - this.kafkaSettings = kafkaSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - - this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); - requestBuilder.admin(transportApiAdmin); - - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); - responseBuilder.clientId("transport-api-response-" + serviceInfoProvider.getServiceId()); - responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - responseBuilder.admin(transportApiAdmin); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(requestBuilder.build()); - templateBuilder.responseTemplate(responseBuilder.build()); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-node-rule-engine-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); - requestBuilder.admin(ruleEngineAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - requestBuilder.admin(coreAdmin); - return requestBuilder.build(); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); - responseBuilder.clientId("transport-api-notifications-" + serviceInfoProvider.getServiceId()); - responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - responseBuilder.admin(notificationAdmin); - return responseBuilder.build(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java deleted file mode 100644 index af0b276c1a..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") -public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); - this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); - this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); - - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java deleted file mode 100644 index 0a23d58e46..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") -public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); - this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java deleted file mode 100644 index 79501130ed..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") -public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - - public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); - this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java deleted file mode 100644 index 7e859e02ab..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class PubSubTransportQueueFactory implements TbTransportQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); - this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic()); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, - transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(producer); - templateBuilder.responseTemplate(consumer); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java deleted file mode 100644 index 72f45e5276..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ /dev/null @@ -1,169 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") -public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbRabbitMqSettings rabbitMqSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbRabbitMqSettings rabbitMqSettings, - TbRabbitMqQueueArguments queueArguments) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.rabbitMqSettings = rabbitMqSettings; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); - this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java deleted file mode 100644 index d84bbedbd2..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") -public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbRabbitMqQueueArguments queueArguments) { - this.rabbitMqSettings = rabbitMqSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); - this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java deleted file mode 100644 index a18f427fab..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") -public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbRabbitMqSettings rabbitMqSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - - public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbRabbitMqSettings rabbitMqSettings, - TbRabbitMqQueueArguments queueArguments) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.rabbitMqSettings = rabbitMqSettings; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java deleted file mode 100644 index 841e004b3f..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -import javax.annotation.PreDestroy; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbRabbitMqSettings rabbitMqSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbRabbitMqSettings rabbitMqSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbRabbitMqQueueArguments queueArguments) { - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.rabbitMqSettings = rabbitMqSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbRabbitMqProducerTemplate> producerTemplate = - new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); - - TbRabbitMqConsumerTemplate> consumerTemplate = - new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, - transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(producerTemplate); - templateBuilder.responseTemplate(consumerTemplate); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java deleted file mode 100644 index 8b01c38607..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") -public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; - - public ServiceBusMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceBusSettings serviceBusSettings, - TbQueueAdmin admin) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.serviceBusSettings = serviceBusSettings; - this.admin = admin; - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java deleted file mode 100644 index 523ac88bd1..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") -public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueAdmin admin; - - public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { - this.serviceBusSettings = serviceBusSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.partitionService = partitionService; - this.serviceInfoProvider = serviceInfoProvider; - this.admin = admin; - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java deleted file mode 100644 index 8a2f9b396b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") -public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final PartitionService partitionService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; - - public ServiceBusTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbServiceBusSettings serviceBusSettings, - TbQueueAdmin admin) { - this.partitionService = partitionService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.serviceBusSettings = serviceBusSettings; - this.admin = admin; - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java deleted file mode 100644 index 33c5277ca5..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") -@Slf4j -public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory { - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - - public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceBusSettings serviceBusSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueAdmin admin) { - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.serviceBusSettings = serviceBusSettings; - this.admin = admin; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producerTemplate = - new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); - - TbQueueConsumer> consumerTemplate = - new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); - templateBuilder.requestTemplate(producerTemplate); - templateBuilder.responseTemplate(consumerTemplate); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, - transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java deleted file mode 100644 index 59a8f449b0..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -/** - * Responsible for initialization of various Producers and Consumers used by TB Core Node. - * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable - */ -public interface TbCoreQueueFactory { - - /** - * Used to push messages to instances of TB Transport Service - * - * @return - */ - TbQueueProducer> createTransportNotificationsMsgProducer(); - - /** - * Used to push messages to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> createRuleEngineMsgProducer(); - - /** - * Used to push notifications to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> createRuleEngineNotificationsMsgProducer(); - - /** - * Used to push messages to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> createTbCoreMsgProducer(); - - /** - * Used to push notifications to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> createTbCoreNotificationsMsgProducer(); - - /** - * Used to consume messages by TB Core Service - * - * @return - */ - TbQueueConsumer> createToCoreMsgConsumer(); - - /** - * Used to consume high priority messages by TB Core Service - * - * @return - */ - TbQueueConsumer> createToCoreNotificationsMsgConsumer(); - - /** - * Used to consume Transport API Calls - * - * @return - */ - TbQueueConsumer> createTransportApiRequestConsumer(); - - /** - * Used to push replies to Transport API Calls - * - * @return - */ - TbQueueProducer> createTransportApiResponseProducer(); - - TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java deleted file mode 100644 index 9fe5de0bcc..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import javax.annotation.PostConstruct; - -@Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") -public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { - - private final TbCoreQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - private TbQueueProducer> toRuleEngineNotifications; - private TbQueueProducer> toTbCoreNotifications; - - public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { - this.tbQueueProvider = tbQueueProvider; - } - - @PostConstruct - public void init() { - this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); - this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); - this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); - this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); - this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); - } - - @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { - return toTransport; - } - - @Override - public TbQueueProducer> getRuleEngineMsgProducer() { - return toRuleEngine; - } - - @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { - return toRuleEngineNotifications; - } - - @Override - public TbQueueProducer> getTbCoreMsgProducer() { - return toTbCore; - } - - @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { - return toTbCoreNotifications; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java deleted file mode 100644 index 34c6ee577f..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -/** - * Responsible for providing various Producers to other services. - */ -public interface TbQueueProducerProvider { - - /** - * Used to push messages to instances of TB Transport Service - * - * @return - */ - TbQueueProducer> getTransportNotificationsMsgProducer(); - - /** - * Used to push messages to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> getRuleEngineMsgProducer(); - - /** - * Used to push notifications to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> getRuleEngineNotificationsMsgProducer(); - - /** - * Used to push messages to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> getTbCoreMsgProducer(); - - /** - * Used to push messages to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> getTbCoreNotificationsMsgProducer(); - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java deleted file mode 100644 index 69de7ca3e2..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import javax.annotation.PostConstruct; - -@Service -@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") -public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { - - private final TbRuleEngineQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - private TbQueueProducer> toRuleEngineNotifications; - private TbQueueProducer> toTbCoreNotifications; - - - public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { - this.tbQueueProvider = tbQueueProvider; - } - - @PostConstruct - public void init() { - this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); - this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); - this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); - this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); - this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); - } - - @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { - return toTransport; - } - - @Override - public TbQueueProducer> getRuleEngineMsgProducer() { - return toRuleEngine; - } - - @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { - return toRuleEngineNotifications; - } - - @Override - public TbQueueProducer> getTbCoreMsgProducer() { - return toTbCore; - } - - @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { - return toTbCoreNotifications; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java deleted file mode 100644 index 561b3e84ea..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; - -/** - * Responsible for initialization of various Producers and Consumers used by TB Core Node. - * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable - */ -public interface TbRuleEngineQueueFactory { - - /** - * Used to push messages to instances of TB Transport Service - * - * @return - */ - TbQueueProducer> createTransportNotificationsMsgProducer(); - - /** - * Used to push messages to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> createRuleEngineMsgProducer(); - - /** - * Used to push notifications to instances of TB RuleEngine Service - * - * @return - */ - TbQueueProducer> createRuleEngineNotificationsMsgProducer(); - - /** - * Used to push messages to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> createTbCoreMsgProducer(); - - /** - * Used to push notifications to other instances of TB Core Service - * - * @return - */ - TbQueueProducer> createTbCoreNotificationsMsgProducer(); - - /** - * Used to consume messages by TB Core Service - * - * @return - * @param configuration - */ - //TODO 2.5 ybondarenko: make sure you use queueName to distinct consumers where necessary - TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration); - - /** - * Used to consume high priority messages by TB Core Service - * - * @return - */ - TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); - - TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java deleted file mode 100644 index dc1d2c449c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; - -public interface TbTransportQueueFactory { - - TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate(); - - TbQueueProducer> createRuleEngineMsgProducer(); - - TbQueueProducer> createTbCoreMsgProducer(); - - TbQueueConsumer> createTransportNotificationsConsumer(); - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java deleted file mode 100644 index bced3f83ac..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -import javax.annotation.PostConstruct; - -@Service -@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") -public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { - - private final TbTransportQueueFactory tbQueueProvider; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - - public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { - this.tbQueueProvider = tbQueueProvider; - } - - @PostConstruct - public void init() { - this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); - this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); - } - - @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Transport!"); - } - - @Override - public TbQueueProducer> getRuleEngineMsgProducer() { - return toRuleEngine; - } - - @Override - public TbQueueProducer> getTbCoreMsgProducer() { - return toTbCore; - } - - @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Transport!"); - } - - @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Transport!"); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java deleted file mode 100644 index 1241370d05..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ /dev/null @@ -1,186 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.cloud.pubsub.v1.SubscriptionAdminClient; -import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; -import com.google.cloud.pubsub.v1.TopicAdminClient; -import com.google.cloud.pubsub.v1.TopicAdminSettings; -import com.google.protobuf.Duration; -import com.google.pubsub.v1.ListSubscriptionsRequest; -import com.google.pubsub.v1.ListTopicsRequest; -import com.google.pubsub.v1.ProjectName; -import com.google.pubsub.v1.ProjectSubscriptionName; -import com.google.pubsub.v1.ProjectTopicName; -import com.google.pubsub.v1.Subscription; -import com.google.pubsub.v1.Topic; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.io.IOException; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -@Slf4j -public class TbPubSubAdmin implements TbQueueAdmin { - private static final String ACK_DEADLINE = "ackDeadlineInSec"; - private static final String MESSAGE_RETENTION = "messageRetentionInSec"; - - private final TopicAdminClient topicAdminClient; - private final SubscriptionAdminClient subscriptionAdminClient; - - private final TbPubSubSettings pubSubSettings; - private final Set topicSet = ConcurrentHashMap.newKeySet(); - private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); - private final Map subscriptionProperties; - - public TbPubSubAdmin(TbPubSubSettings pubSubSettings, Map subscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.subscriptionProperties = subscriptionSettings; - - TopicAdminSettings topicAdminSettings; - try { - topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); - } catch (IOException e) { - log.error("Failed to create TopicAdminSettings"); - throw new RuntimeException("Failed to create TopicAdminSettings."); - } - - SubscriptionAdminSettings subscriptionAdminSettings; - try { - subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); - } catch (IOException e) { - log.error("Failed to create SubscriptionAdminSettings"); - throw new RuntimeException("Failed to create SubscriptionAdminSettings."); - } - - try { - topicAdminClient = TopicAdminClient.create(topicAdminSettings); - - ListTopicsRequest listTopicsRequest = - ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); - TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); - for (Topic topic : response.iterateAll()) { - topicSet.add(topic.getName()); - } - } catch (IOException e) { - log.error("Failed to get topics.", e); - throw new RuntimeException("Failed to get topics.", e); - } - - try { - subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings); - - ListSubscriptionsRequest listSubscriptionsRequest = - ListSubscriptionsRequest.newBuilder() - .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) - .build(); - SubscriptionAdminClient.ListSubscriptionsPagedResponse response = - subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); - - for (Subscription subscription : response.iterateAll()) { - subscriptionSet.add(subscription.getName()); - } - } catch (IOException e) { - log.error("Failed to get subscriptions.", e); - throw new RuntimeException("Failed to get subscriptions.", e); - } - } - - @Override - public void createTopicIfNotExists(String partition) { - ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), partition); - - if (topicSet.contains(topicName.toString())) { - createSubscriptionIfNotExists(partition, topicName); - return; - } - - ListTopicsRequest listTopicsRequest = - ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); - TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); - for (Topic topic : response.iterateAll()) { - if (topic.getName().contains(topicName.toString())) { - topicSet.add(topic.getName()); - createSubscriptionIfNotExists(partition, topicName); - return; - } - } - - topicAdminClient.createTopic(topicName); - topicSet.add(topicName.toString()); - log.info("Created new topic: [{}]", topicName.toString()); - createSubscriptionIfNotExists(partition, topicName); - } - - private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { - ProjectSubscriptionName subscriptionName = - ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); - - if (subscriptionSet.contains(subscriptionName.toString())) { - return; - } - - ListSubscriptionsRequest listSubscriptionsRequest = - ListSubscriptionsRequest.newBuilder().setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()).build(); - SubscriptionAdminClient.ListSubscriptionsPagedResponse response = subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); - for (Subscription subscription : response.iterateAll()) { - if (subscription.getName().equals(subscriptionName.toString())) { - subscriptionSet.add(subscription.getName()); - return; - } - } - - Subscription.Builder subscriptionBuilder = Subscription - .newBuilder() - .setName(subscriptionName.toString()) - .setTopic(topicName.toString()); - - setAckDeadline(subscriptionBuilder); - setMessageRetention(subscriptionBuilder); - - subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); - subscriptionSet.add(subscriptionName.toString()); - log.info("Created new subscription: [{}]", subscriptionName.toString()); - } - - private void setAckDeadline(Subscription.Builder builder) { - if (subscriptionProperties.containsKey(ACK_DEADLINE)) { - builder.setAckDeadlineSeconds(Integer.parseInt(subscriptionProperties.get(ACK_DEADLINE))); - } - } - - private void setMessageRetention(Subscription.Builder builder) { - if (subscriptionProperties.containsKey(MESSAGE_RETENTION)) { - Duration duration = Duration - .newBuilder() - .setSeconds(Long.parseLong(subscriptionProperties.get(MESSAGE_RETENTION))) - .build(); - builder.setMessageRetentionDuration(duration); - } - } - - @Override - public void destroy() { - if (topicAdminClient != null) { - topicAdminClient.close(); - } - if (subscriptionAdminClient != null) { - subscriptionAdminClient.close(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java deleted file mode 100644 index 5dc795739a..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ /dev/null @@ -1,219 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.api.core.ApiFuture; -import com.google.api.core.ApiFutures; -import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; -import com.google.cloud.pubsub.v1.stub.SubscriberStub; -import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings; -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.pubsub.v1.AcknowledgeRequest; -import com.google.pubsub.v1.ProjectSubscriptionName; -import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.v1.PullRequest; -import com.google.pubsub.v1.PullResponse; -import com.google.pubsub.v1.ReceivedMessage; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -@Slf4j -public class TbPubSubConsumerTemplate implements TbQueueConsumer { - - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final String topic; - private final TbQueueMsgDecoder decoder; - private final TbPubSubSettings pubSubSettings; - - private volatile boolean subscribed; - private volatile Set partitions; - private volatile Set subscriptionNames; - private final List acknowledgeRequests = new CopyOnWriteArrayList<>(); - - private ExecutorService consumerExecutor; - private final SubscriberStub subscriber; - private volatile boolean stopped; - - private volatile int messagesPerTopic; - - public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { - this.admin = admin; - this.pubSubSettings = pubSubSettings; - this.topic = topic; - this.decoder = decoder; - - try { - SubscriberStubSettings subscriberStubSettings = - SubscriberStubSettings.newBuilder() - .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) - .setTransportChannelProvider( - SubscriberStubSettings.defaultGrpcTransportProviderBuilder() - .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) - .build()) - .build(); - - this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); - } catch (IOException e) { - log.error("Failed to create subscriber.", e); - throw new RuntimeException("Failed to create subscriber.", e); - } - stopped = false; - } - - @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public void unsubscribe() { - stopped = true; - if (consumerExecutor != null) { - consumerExecutor.shutdownNow(); - } - - if (subscriber != null) { - subscriber.close(); - } - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); - subscriptionNames.forEach(admin::createTopicIfNotExists); - consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); - messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); - subscribed = true; - } - List messages; - try { - messages = receiveMessages(); - if (!messages.isEmpty()) { - List result = new ArrayList<>(); - messages.forEach(msg -> { - try { - result.add(decode(msg.getMessage())); - } catch (InvalidProtocolBufferException e) { - log.error("Failed decode record: [{}]", msg); - } - }); - return result; - } - } catch (ExecutionException | InterruptedException e) { - if (stopped) { - log.info("[{}] Pub/Sub consumer is stopped.", topic); - } else { - log.error("Failed to receive messages", e); - } - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); - acknowledgeRequests.clear(); - } - - private List receiveMessages() throws ExecutionException, InterruptedException { - List>> result = subscriptionNames.stream().map(subscriptionId -> { - String subscriptionName = ProjectSubscriptionName.format(pubSubSettings.getProjectId(), subscriptionId); - PullRequest pullRequest = - PullRequest.newBuilder() - .setMaxMessages(messagesPerTopic) - .setReturnImmediately(false) // return immediately if messages are not available - .setSubscription(subscriptionName) - .build(); - - ApiFuture pullResponseApiFuture = subscriber.pullCallable().futureCall(pullRequest); - - return ApiFutures.transform(pullResponseApiFuture, pullResponse -> { - if (pullResponse != null && !pullResponse.getReceivedMessagesList().isEmpty()) { - List ackIds = new ArrayList<>(); - for (ReceivedMessage message : pullResponse.getReceivedMessagesList()) { - ackIds.add(message.getAckId()); - } - AcknowledgeRequest acknowledgeRequest = - AcknowledgeRequest.newBuilder() - .setSubscription(subscriptionName) - .addAllAckIds(ackIds) - .build(); - - acknowledgeRequests.add(acknowledgeRequest); - return pullResponse.getReceivedMessagesList(); - } - return null; - }, consumerExecutor); - - }).collect(Collectors.toList()); - - ApiFuture> transform = ApiFutures.transform(ApiFutures.allAsList(result), listMessages -> { - if (!CollectionUtils.isEmpty(listMessages)) { - return listMessages.stream().filter(Objects::nonNull).flatMap(List::stream).collect(Collectors.toList()); - } - return Collections.emptyList(); - }, consumerExecutor); - - return transform.get(); - } - - public T decode(PubsubMessage message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java deleted file mode 100644 index 2cd2e1054e..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.api.core.ApiFuture; -import com.google.api.core.ApiFutureCallback; -import com.google.api.core.ApiFutures; -import com.google.cloud.pubsub.v1.Publisher; -import com.google.gson.Gson; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.ProjectTopicName; -import com.google.pubsub.v1.PubsubMessage; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -@Slf4j -public class TbPubSubProducerTemplate implements TbQueueProducer { - - private final Gson gson = new Gson(); - - private final String defaultTopic; - private final TbQueueAdmin admin; - private final TbPubSubSettings pubSubSettings; - - private final Map publisherMap = new ConcurrentHashMap<>(); - - private ExecutorService pubExecutor = Executors.newCachedThreadPool(); - - public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) { - this.defaultTopic = defaultTopic; - this.admin = admin; - this.pubSubSettings = pubSubSettings; - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - PubsubMessage.Builder pubsubMessageBuilder = PubsubMessage.newBuilder(); - pubsubMessageBuilder.setData(getMsg(msg)); - - Publisher publisher = getOrCreatePublisher(tpi.getFullTopicName()); - ApiFuture future = publisher.publish(pubsubMessageBuilder.build()); - - ApiFutures.addCallback(future, new ApiFutureCallback() { - public void onSuccess(String messageId) { - if (callback != null) { - callback.onSuccess(null); - } - } - - public void onFailure(Throwable t) { - if (callback != null) { - callback.onFailure(t); - } - } - }, pubExecutor); - } - - @Override - public void stop() { - publisherMap.forEach((k, v) -> { - if (v != null) { - try { - v.shutdown(); - v.awaitTermination(1, TimeUnit.SECONDS); - } catch (Exception e) { - log.error("Failed to shutdown PubSub client during destroy()", e); - } - } - }); - - if (pubExecutor != null) { - pubExecutor.shutdownNow(); - } - } - - private ByteString getMsg(T msg) { - String json = gson.toJson(new DefaultTbQueueMsg(msg)); - return ByteString.copyFrom(json.getBytes()); - } - - private Publisher getOrCreatePublisher(String topic) { - if (publisherMap.containsKey(topic)) { - return publisherMap.get(topic); - } else { - try { - admin.createTopicIfNotExists(topic); - ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), topic); - Publisher publisher = Publisher.newBuilder(topicName).setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); - publisherMap.put(topic, publisher); - return publisher; - } catch (IOException e) { - log.error("Failed to create topic [{}].", topic, e); - throw new RuntimeException("Failed to create topic.", e); - } - } - - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java deleted file mode 100644 index d6c6d0e271..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.api.gax.core.CredentialsProvider; -import com.google.api.gax.core.FixedCredentialsProvider; -import com.google.auth.oauth2.ServiceAccountCredentials; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.io.ByteArrayInputStream; -import java.io.IOException; - -@Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") -@Component -@Data -public class TbPubSubSettings { - - @Value("${queue.pubsub.project_id}") - private String projectId; - - @Value("${queue.pubsub.service_account}") - private String serviceAccount; - - @Value("${queue.pubsub.max_msg_size}") - private int maxMsgSize; - - @Value("${queue.pubsub.max_messages}") - private int maxMessages; - - private CredentialsProvider credentialsProvider; - - @PostConstruct - private void init() throws IOException { - ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream( - new ByteArrayInputStream(serviceAccount.getBytes())); - credentialsProvider = FixedCredentialsProvider.create(credentials); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java deleted file mode 100644 index e98f51ec7c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") -public class TbPubSubSubscriptionSettings { - @Value("${queue.pubsub.queue-properties.core}") - private String coreProperties; - @Value("${queue.pubsub.queue-properties.rule-engine}") - private String ruleEngineProperties; - @Value("${queue.pubsub.queue-properties.transport-api}") - private String transportApiProperties; - @Value("${queue.pubsub.queue-properties.notifications}") - private String notificationsProperties; - @Value("${queue.pubsub.queue-properties.js-executor}") - private String jsExecutorProperties; - - @Getter - private Map coreSettings; - @Getter - private Map ruleEngineSettings; - @Getter - private Map transportApiSettings; - @Getter - private Map notificationsSettings; - @Getter - private Map jsExecutorSettings; - - @PostConstruct - private void init() { - coreSettings = getSettings(coreProperties); - ruleEngineSettings = getSettings(ruleEngineProperties); - transportApiSettings = getSettings(transportApiProperties); - notificationsSettings = getSettings(notificationsProperties); - jsExecutorSettings = getSettings(jsExecutorProperties); - } - - private Map getSettings(String properties) { - Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - configs.put(key, value); - } - return configs; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java deleted file mode 100644 index 3bef6deb84..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.TimeoutException; - -@Slf4j -public class TbRabbitMqAdmin implements TbQueueAdmin { - - private final TbRabbitMqSettings rabbitMqSettings; - private final Channel channel; - private final Connection connection; - private final Map arguments; - - public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map arguments) { - this.rabbitMqSettings = rabbitMqSettings; - this.arguments = arguments; - - try { - connection = rabbitMqSettings.getConnectionFactory().newConnection(); - } catch (IOException | TimeoutException e) { - log.error("Failed to create connection.", e); - throw new RuntimeException("Failed to create connection.", e); - } - - try { - channel = connection.createChannel(); - } catch (IOException e) { - log.error("Failed to create chanel.", e); - throw new RuntimeException("Failed to create chanel.", e); - } - } - - @Override - public void createTopicIfNotExists(String topic) { - try { - channel.queueDeclare(topic, false, false, false, arguments); - } catch (IOException e) { - log.error("Failed to bind queue: [{}]", topic, e); - } - } - - @Override - public void destroy() { - if (channel != null) { - try { - channel.close(); - } catch (IOException | TimeoutException e) { - log.error("Failed to close Chanel.", e); - } - } - if (connection != null) { - try { - connection.close(); - } catch (IOException e) { - log.error("Failed to close Connection.", e); - } - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java deleted file mode 100644 index 25d7719163..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.GetResponse; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; - -@Slf4j -public class TbRabbitMqConsumerTemplate implements TbQueueConsumer { - - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final String topic; - private final TbQueueMsgDecoder decoder; - private final TbRabbitMqSettings rabbitMqSettings; - private final Channel channel; - private final Connection connection; - - private volatile Set partitions; - private volatile boolean subscribed; - private volatile Set queues; - private volatile boolean stopped; - - public TbRabbitMqConsumerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String topic, TbQueueMsgDecoder decoder) { - this.admin = admin; - this.decoder = decoder; - this.topic = topic; - this.rabbitMqSettings = rabbitMqSettings; - try { - connection = rabbitMqSettings.getConnectionFactory().newConnection(); - } catch (IOException | TimeoutException e) { - log.error("Failed to create connection.", e); - throw new RuntimeException("Failed to create connection.", e); - } - - try { - channel = connection.createChannel(); - } catch (IOException e) { - log.error("Failed to create chanel.", e); - throw new RuntimeException("Failed to create chanel.", e); - } - stopped = false; - } - - @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public void unsubscribe() { - stopped = true; - if (channel != null) { - try { - channel.close(); - } catch (IOException | TimeoutException e) { - log.error("Failed to close the channel."); - } - } - if (connection != null) { - try { - connection.close(); - } catch (IOException e) { - log.error("Failed to close the connection."); - } - } - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - queues = partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .collect(Collectors.toSet()); - - queues.forEach(admin::createTopicIfNotExists); - subscribed = true; - } - - List result = queues.stream() - .map(queue -> { - try { - return channel.basicGet(queue, false); - } catch (IOException e) { - log.error("Failed to get messages from queue: [{}]", queue); - throw new RuntimeException("Failed to get messages from queue.", e); - } - }).filter(Objects::nonNull).map(message -> { - try { - return decode(message); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to decode message: [{}].", message); - throw new RuntimeException("Failed to decode message.", e); - } - }).collect(Collectors.toList()); - if (result.size() > 0) { - return result; - } - } - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - if (!stopped) { - log.error("Failed to wait.", e); - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - try { - channel.basicAck(0, true); - } catch (IOException e) { - log.error("Failed to ack messages.", e); - } - } - - public T decode(GetResponse message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(new String(message.getBody()), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java deleted file mode 100644 index 91b46213a5..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.Gson; -import com.rabbitmq.client.AMQP; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeoutException; - -@Slf4j -public class TbRabbitMqProducerTemplate implements TbQueueProducer { - private final String defaultTopic; - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final TbRabbitMqSettings rabbitMqSettings; - private ListeningExecutorService producerExecutor; - private final Channel channel; - private final Connection connection; - - public TbRabbitMqProducerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String defaultTopic) { - this.admin = admin; - this.defaultTopic = defaultTopic; - this.rabbitMqSettings = rabbitMqSettings; - producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); - try { - connection = rabbitMqSettings.getConnectionFactory().newConnection(); - } catch (IOException | TimeoutException e) { - log.error("Failed to create connection.", e); - throw new RuntimeException("Failed to create connection.", e); - } - - try { - channel = connection.createChannel(); - } catch (IOException e) { - log.error("Failed to create chanel.", e); - throw new RuntimeException("Failed to create chanel.", e); - } - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - AMQP.BasicProperties properties = new AMQP.BasicProperties(); - try { - channel.basicPublish(rabbitMqSettings.getExchangeName(), tpi.getFullTopicName(), properties, gson.toJson(new DefaultTbQueueMsg(msg)).getBytes()); - if (callback != null) { - callback.onSuccess(null); - } - } catch (IOException e) { - log.error("Failed publish message: [{}].", msg, e); - if (callback != null) { - callback.onFailure(e); - } - } - } - - @Override - public void stop() { - if (producerExecutor != null) { - producerExecutor.shutdownNow(); - } - if (channel != null) { - try { - channel.close(); - } catch (IOException | TimeoutException e) { - log.error("Failed to close the channel."); - } - } - if (connection != null) { - try { - connection.close(); - } catch (IOException e) { - log.error("Failed to close the connection."); - } - } - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java deleted file mode 100644 index eb73e7dce6..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Pattern; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") -public class TbRabbitMqQueueArguments { - @Value("${queue.rabbitmq.queue-properties.core}") - private String coreProperties; - @Value("${queue.rabbitmq.queue-properties.rule-engine}") - private String ruleEngineProperties; - @Value("${queue.rabbitmq.queue-properties.transport-api}") - private String transportApiProperties; - @Value("${queue.rabbitmq.queue-properties.notifications}") - private String notificationsProperties; - @Value("${queue.rabbitmq.queue-properties.js-executor}") - private String jsExecutorProperties; - - @Getter - private Map coreArgs; - @Getter - private Map ruleEngineArgs; - @Getter - private Map transportApiArgs; - @Getter - private Map notificationsArgs; - @Getter - private Map jsExecutorArgs; - - @PostConstruct - private void init() { - coreArgs = getArgs(coreProperties); - ruleEngineArgs = getArgs(ruleEngineProperties); - transportApiArgs = getArgs(transportApiProperties); - notificationsArgs = getArgs(notificationsProperties); - jsExecutorArgs = getArgs(jsExecutorProperties); - } - - private Map getArgs(String properties) { - Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String strValue = property.substring(delimiterPosition + 1); - configs.put(key, getObjectValue(strValue)); - } - return configs; - } - - private Object getObjectValue(String str) { - if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) { - return Boolean.valueOf(str); - } else if (isNumeric(str)) { - return getNumericValue(str); - } - return str; - } - - private Object getNumericValue(String str) { - if (str.contains(".")) { - return Double.valueOf(str); - } else { - return Long.valueOf(str); - } - } - - private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?"); - - public boolean isNumeric(String strNum) { - if (strNum == null) { - return false; - } - return PATTERN.matcher(strNum).matches(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java deleted file mode 100644 index e0156e6dc8..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.rabbitmq.client.ConnectionFactory; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; - -@Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") -@Component -@Data -public class TbRabbitMqSettings { - @Value("${queue.rabbitmq.exchange_name:}") - private String exchangeName; - @Value("${queue.rabbitmq.host:}") - private String host; - @Value("${queue.rabbitmq.port:}") - private int port; - @Value("${queue.rabbitmq.virtual_host:}") - private String virtualHost; - @Value("${queue.rabbitmq.username:}") - private String username; - @Value("${queue.rabbitmq.password:}") - private String password; - @Value("${queue.rabbitmq.automatic_recovery_enabled:}") - private boolean automaticRecoveryEnabled; - @Value("${queue.rabbitmq.connection_timeout:}") - private int connectionTimeout; - @Value("${queue.rabbitmq.handshake_timeout:}") - private int handshakeTimeout; - - private ConnectionFactory connectionFactory; - - @PostConstruct - private void init() { - connectionFactory = new ConnectionFactory(); - connectionFactory.setHost(host); - connectionFactory.setPort(port); - connectionFactory.setVirtualHost(virtualHost); - connectionFactory.setUsername(username); - connectionFactory.setPassword(password); - connectionFactory.setAutomaticRecoveryEnabled(automaticRecoveryEnabled); - connectionFactory.setConnectionTimeout(connectionTimeout); - connectionFactory.setHandshakeTimeout(handshakeTimeout); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java deleted file mode 100644 index 1c260c754d..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.settings; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -public class TbQueueCoreSettings { - - @Value("${queue.core.topic}") - private String topic; - - @Value("${queue.core.partitions}") - private int partitions; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java deleted file mode 100644 index cd492e87f1..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.settings; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -public class TbQueueRemoteJsInvokeSettings { - @Value("${queue.js.request_topic}") - private String requestTopic; - - @Value("${queue.js.response_topic_prefix}") - private String responseTopic; - - @Value("${queue.js.max_pending_requests}") - private long maxPendingRequests; - - @Value("${queue.js.response_poll_interval}") - private int responsePollInterval; - - @Value("${queue.js.response_auto_commit_interval}") - private int autoCommitInterval; - - @Value("${queue.js.max_requests_timeout}") - private long maxRequestsTimeout; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java deleted file mode 100644 index 9dfe715ece..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.settings; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -import javax.annotation.PostConstruct; -import java.util.List; - -@Slf4j -@Data -@EnableAutoConfiguration -@Configuration -@ConfigurationProperties(prefix = "queue.rule-engine") -public class TbQueueRuleEngineSettings { - - private String topic; - private List queues; - - //TODO 2.5 ybondarenko: make sure the queue names are valid to all queue providers. - // See how they are used in TbRuleEngineQueueFactory.createToRuleEngineMsgConsumer and all producers - @PostConstruct - public void validate() { - queues.stream().filter(queue -> queue.getName().equals("Main")).findFirst().orElseThrow(() -> { - log.error("Main queue is not configured in thingsboard.yml"); - return new RuntimeException("No \"Main\" queue configured!"); - }); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java deleted file mode 100644 index 5ee58ab17e..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.settings; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -public class TbQueueTransportApiSettings { - - @Value("${queue.transport_api.requests_topic}") - private String requestsTopic; - - @Value("${queue.transport_api.responses_topic}") - private String responsesTopic; - - @Value("${queue.transport_api.max_pending_requests}") - private int maxPendingRequests; - - @Value("${queue.transport_api.max_requests_timeout}") - private int maxRequestsTimeout; - - @Value("${queue.transport_api.max_callback_threads}") - private int maxCallbackThreads; - - @Value("${queue.transport_api.request_poll_interval}") - private long requestPollInterval; - - @Value("${queue.transport_api.response_poll_interval}") - private long responsePollInterval; - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java deleted file mode 100644 index 019a76953b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.settings; - -import lombok.Data; - -@Data -public class TbRuleEngineQueueConfiguration { - - private String name; - private String topic; - private int pollInterval; - private int partitions; - private long packProcessingTimeout; - private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy; - private TbRuleEngineQueueAckStrategyConfiguration processingStrategy; - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java deleted file mode 100644 index eb4eaed7ed..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.http.SdkHttpMetadata; -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.queue.TbQueueMsgMetadata; - -@Data -@AllArgsConstructor -public class AwsSqsTbQueueMsgMetadata implements TbQueueMsgMetadata { - - private final SdkHttpMetadata metadata; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java deleted file mode 100644 index 8e293e6dff..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.sqs.AmazonSQS; -import com.amazonaws.services.sqs.AmazonSQSClientBuilder; -import com.amazonaws.services.sqs.model.CreateQueueRequest; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -public class TbAwsSqsAdmin implements TbQueueAdmin { - - private final Map attributes; - private final AmazonSQS sqsClient; - private final Set queues; - - public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings, Map attributes) { - this.attributes = attributes; - - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(new AWSStaticCredentialsProvider(awsCredentials)) - .withRegion(sqsSettings.getRegion()) - .build(); - - queues = sqsClient - .listQueues() - .getQueueUrls() - .stream() - .map(this::getQueueNameFromUrl) - .collect(Collectors.toCollection(ConcurrentHashMap::newKeySet)); - } - - @Override - public void createTopicIfNotExists(String topic) { - String queueName = topic.replaceAll("\\.", "_") + ".fifo"; - if (queues.contains(queueName)) { - return; - } - final CreateQueueRequest createQueueRequest = new CreateQueueRequest(queueName).withAttributes(attributes); - String queueUrl = sqsClient.createQueue(createQueueRequest).getQueueUrl(); - queues.add(getQueueNameFromUrl(queueUrl)); - } - - private String getQueueNameFromUrl(String queueUrl) { - int delimiterIndex = queueUrl.lastIndexOf("/"); - return queueUrl.substring(delimiterIndex + 1); - } - - @Override - public void destroy() { - if (sqsClient != null) { - sqsClient.shutdown(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java deleted file mode 100644 index 3e71388844..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ /dev/null @@ -1,231 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.sqs.AmazonSQS; -import com.amazonaws.services.sqs.AmazonSQSClientBuilder; -import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; -import com.amazonaws.services.sqs.model.Message; -import com.amazonaws.services.sqs.model.ReceiveMessageRequest; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -@Slf4j -public class TbAwsSqsConsumerTemplate implements TbQueueConsumer { - - private static final int MAX_NUM_MSGS = 10; - - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final AmazonSQS sqsClient; - private final String topic; - private final TbQueueMsgDecoder decoder; - private final TbAwsSqsSettings sqsSettings; - - private final List pendingMessages = new CopyOnWriteArrayList<>(); - private volatile Set queueUrls; - private volatile Set partitions; - private ListeningExecutorService consumerExecutor; - private volatile boolean subscribed; - private volatile boolean stopped = false; - - public TbAwsSqsConsumerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String topic, TbQueueMsgDecoder decoder) { - this.admin = admin; - this.decoder = decoder; - this.topic = topic; - this.sqsSettings = sqsSettings; - - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); - - this.sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(credProvider) - .withRegion(sqsSettings.getRegion()) - .build(); - } - - @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public void unsubscribe() { - stopped = true; - - if (sqsClient != null) { - sqsClient.shutdown(); - } - if (consumerExecutor != null) { - consumerExecutor.shutdownNow(); - } - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); - consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1)); - subscribed = true; - } - - if (!pendingMessages.isEmpty()) { - log.warn("Present {} non committed messages.", pendingMessages.size()); - return Collections.emptyList(); - } - - List>> futureList = queueUrls - .stream() - .map(url -> poll(url, (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis))) - .collect(Collectors.toList()); - ListenableFuture>> futureResult = Futures.allAsList(futureList); - try { - return futureResult.get().stream() - .flatMap(List::stream) - .map(msg -> { - try { - return decode(msg); - } catch (IOException e) { - log.error("Failed to decode message: [{}]", msg); - return null; - } - }).filter(Objects::nonNull) - .collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Aws SQS consumer is stopped.", topic); - } else { - log.error("Failed to pool messages.", e); - } - } - } - return Collections.emptyList(); - } - - private ListenableFuture> poll(String url, int waitTimeSeconds) { - List>> result = new ArrayList<>(); - - for (int i = 0; i < sqsSettings.getThreadsPerTopic(); i++) { - result.add(consumerExecutor.submit(() -> { - ReceiveMessageRequest request = new ReceiveMessageRequest(); - request - .withWaitTimeSeconds(waitTimeSeconds) - .withMessageAttributeNames("headers") - .withQueueUrl(url) - .withMaxNumberOfMessages(MAX_NUM_MSGS); - return sqsClient.receiveMessage(request).getMessages(); - })); - } - return Futures.transform(Futures.allAsList(result), list -> { - if (!CollectionUtils.isEmpty(list)) { - return list.stream() - .flatMap(messageList -> { - if (!messageList.isEmpty()) { - this.pendingMessages.add(new AwsSqsMsgWrapper(url, messageList)); - return messageList.stream(); - } - return Stream.empty(); - }) - .collect(Collectors.toList()); - } - return Collections.emptyList(); - }, consumerExecutor); - } - - @Override - public void commit() { - pendingMessages.forEach(msg -> - consumerExecutor.submit(() -> { - List entries = msg.getMessages() - .stream() - .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) - .collect(Collectors.toList()); - sqsClient.deleteMessageBatch(msg.getUrl(), entries); - })); - - pendingMessages.clear(); - } - - public T decode(Message message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - - @Data - private static class AwsSqsMsgWrapper { - private final String url; - private final List messages; - - public AwsSqsMsgWrapper(String url, List messages) { - this.url = url; - this.messages = messages; - } - } - - private String getQueueUrl(String topic) { - admin.createTopicIfNotExists(topic); - return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java deleted file mode 100644 index 2d85539184..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.sqs.AmazonSQS; -import com.amazonaws.services.sqs.AmazonSQSClientBuilder; -import com.amazonaws.services.sqs.model.SendMessageRequest; -import com.amazonaws.services.sqs.model.SendMessageResult; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.Gson; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; - -@Slf4j -public class TbAwsSqsProducerTemplate implements TbQueueProducer { - private final String defaultTopic; - private final AmazonSQS sqsClient; - private final Gson gson = new Gson(); - private final Map queueUrlMap = new ConcurrentHashMap<>(); - private final TbQueueAdmin admin; - private ListeningExecutorService producerExecutor; - - public TbAwsSqsProducerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String defaultTopic) { - this.admin = admin; - this.defaultTopic = defaultTopic; - - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); - - this.sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(credProvider) - .withRegion(sqsSettings.getRegion()) - .build(); - - producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - SendMessageRequest sendMsgRequest = new SendMessageRequest(); - sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); - sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg))); - - sendMsgRequest.withMessageGroupId(msg.getKey().toString()); - ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); - - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(SendMessageResult result) { - if (callback != null) { - callback.onSuccess(new AwsSqsTbQueueMsgMetadata(result.getSdkHttpMetadata())); - } - } - - @Override - public void onFailure(Throwable t) { - if (callback != null) { - callback.onFailure(t); - } - } - }, producerExecutor); - } - - @Override - public void stop() { - if (producerExecutor != null) { - producerExecutor.shutdownNow(); - } - if (sqsClient != null) { - sqsClient.shutdown(); - } - } - - private String getQueueUrl(String topic) { - return queueUrlMap.computeIfAbsent(topic, k -> { - admin.createTopicIfNotExists(topic); - return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); - }); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java deleted file mode 100644 index 70a9587bef..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.services.sqs.model.QueueAttributeName; -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") -public class TbAwsSqsQueueAttributes { - @Value("${queue.aws-sqs.queue-properties.core}") - private String coreProperties; - @Value("${queue.aws-sqs.queue-properties.rule-engine}") - private String ruleEngineProperties; - @Value("${queue.aws-sqs.queue-properties.transport-api}") - private String transportApiProperties; - @Value("${queue.aws-sqs.queue-properties.notifications}") - private String notificationsProperties; - @Value("${queue.aws-sqs.queue-properties.js-executor}") - private String jsExecutorProperties; - - @Getter - private Map coreAttributes; - @Getter - private Map ruleEngineAttributes; - @Getter - private Map transportApiAttributes; - @Getter - private Map notificationsAttributes; - @Getter - private Map jsExecutorAttributes; - - private final Map defaultAttributes = new HashMap<>(); - - @PostConstruct - private void init() { - defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true"); - defaultAttributes.put(QueueAttributeName.ContentBasedDeduplication.toString(), "true"); - - coreAttributes = getConfigs(coreProperties); - ruleEngineAttributes = getConfigs(ruleEngineProperties); - transportApiAttributes = getConfigs(transportApiProperties); - notificationsAttributes = getConfigs(notificationsProperties); - jsExecutorAttributes = getConfigs(jsExecutorProperties); - } - - private Map getConfigs(String properties) { - Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - validateAttributeName(key); - configs.put(key, value); - } - configs.putAll(defaultAttributes); - return configs; - } - - private void validateAttributeName(String key) { - QueueAttributeName.fromValue(key); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java deleted file mode 100644 index ca2316e174..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") -public @interface TbCoreComponent { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java deleted file mode 100644 index 3b99c94dea..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") -public @interface TbRuleEngineComponent { -} diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto deleted file mode 100644 index 51bc7649be..0000000000 --- a/common/queue/src/main/proto/queue.proto +++ /dev/null @@ -1,410 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; -package transport; - -option java_package = "org.thingsboard.server.gen.transport"; -option java_outer_classname = "TransportProtos"; - -message QueueInfo { - string name = 1; - string topic = 2; - int32 partitions = 3; -} - -/** - * Service Discovery Data Structures; - */ -message ServiceInfo { - string serviceId = 1; - repeated string serviceTypes = 2; - int64 tenantIdMSB = 3; - int64 tenantIdLSB = 4; - repeated QueueInfo ruleEngineQueues = 5; -} - -/** - * Transport Service Data Structures; - */ -message SessionInfoProto { - string nodeId = 1; - int64 sessionIdMSB = 2; - int64 sessionIdLSB = 3; - int64 tenantIdMSB = 4; - int64 tenantIdLSB = 5; - int64 deviceIdMSB = 6; - int64 deviceIdLSB = 7; - string deviceName = 8; - string deviceType = 9; -} - -enum SessionEvent { - OPEN = 0; - CLOSED = 1; -} - -enum SessionType { - SYNC = 0; - ASYNC = 1; -} - -enum KeyValueType { - BOOLEAN_V = 0; - LONG_V = 1; - DOUBLE_V = 2; - STRING_V = 3; - JSON_V = 4; -} - -message KeyValueProto { - string key = 1; - KeyValueType type = 2; - bool bool_v = 3; - int64 long_v = 4; - double double_v = 5; - string string_v = 6; - string json_v = 7; -} - -message TsKvProto { - int64 ts = 1; - KeyValueProto kv = 2; -} - -message TsKvListProto { - int64 ts = 1; - repeated KeyValueProto kv = 2; -} - -message DeviceInfoProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 deviceIdMSB = 3; - int64 deviceIdLSB = 4; - string deviceName = 5; - string deviceType = 6; - string additionalInfo = 7; -} - -/** - * Transport Service Messages; - */ -message SessionEventMsg { - SessionType sessionType = 1; - SessionEvent event = 2; -} - -message PostTelemetryMsg { - repeated TsKvListProto tsKvList = 1; -} - -message PostAttributeMsg { - repeated KeyValueProto kv = 1; -} - -message GetAttributeRequestMsg { - int32 requestId = 1; - repeated string clientAttributeNames = 2; - repeated string sharedAttributeNames = 3; -} - -message GetAttributeResponseMsg { - int32 requestId = 1; - repeated TsKvProto clientAttributeList = 2; - repeated TsKvProto sharedAttributeList = 3; - repeated string deletedAttributeKeys = 4; - string error = 5; -} - -message AttributeUpdateNotificationMsg { - repeated TsKvProto sharedUpdated = 1; - repeated string sharedDeleted = 2; -} - -message ValidateDeviceTokenRequestMsg { - string token = 1; -} - -message ValidateDeviceX509CertRequestMsg { - string hash = 1; -} - -message ValidateDeviceCredentialsResponseMsg { - DeviceInfoProto deviceInfo = 1; - string credentialsBody = 2; -} - -message GetOrCreateDeviceFromGatewayRequestMsg { - int64 gatewayIdMSB = 1; - int64 gatewayIdLSB = 2; - string deviceName = 3; - string deviceType = 4; -} - -message GetOrCreateDeviceFromGatewayResponseMsg { - DeviceInfoProto deviceInfo = 1; -} - -message GetTenantRoutingInfoRequestMsg { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; -} - -message GetTenantRoutingInfoResponseMsg { - bool isolatedTbCore = 1; - bool isolatedTbRuleEngine = 2; -} - -message SessionCloseNotificationProto { - string message = 1; -} - -message SubscribeToAttributeUpdatesMsg { - bool unsubscribe = 1; -} - -message SubscribeToRPCMsg { - bool unsubscribe = 1; -} - -message ToDeviceRpcRequestMsg { - int32 requestId = 1; - string methodName = 2; - string params = 3; -} - -message ToDeviceRpcResponseMsg { - int32 requestId = 1; - string payload = 2; -} - -message ToServerRpcRequestMsg { - int32 requestId = 1; - string methodName = 2; - string params = 3; -} - -message ToServerRpcResponseMsg { - int32 requestId = 1; - string payload = 2; - string error = 3; -} - -message ClaimDeviceMsg { - int64 deviceIdMSB = 1; - int64 deviceIdLSB = 2; - string secretKey = 3; - int64 durationMs = 4; -} - -//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. -message SubscriptionInfoProto { - int64 lastActivityTime = 1; - bool attributeSubscription = 2; - bool rpcSubscription = 3; -} - -message SessionSubscriptionInfoProto { - SessionInfoProto sessionInfo = 1; - SubscriptionInfoProto subscriptionInfo = 2; -} - -message DeviceSessionsCacheEntry { - repeated SessionSubscriptionInfoProto sessions = 1; -} - -message TransportToDeviceActorMsg { - SessionInfoProto sessionInfo = 1; - SessionEventMsg sessionEvent = 2; - GetAttributeRequestMsg getAttributes = 3; - SubscribeToAttributeUpdatesMsg subscribeToAttributes = 4; - SubscribeToRPCMsg subscribeToRPC = 5; - ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 6; - SubscriptionInfoProto subscriptionInfo = 7; - ClaimDeviceMsg claimDevice = 8; -} - -message TransportToRuleEngineMsg { - SessionInfoProto sessionInfo = 1; - PostTelemetryMsg postTelemetry = 2; - PostAttributeMsg postAttributes = 3; - ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 4; - ToServerRpcRequestMsg toServerRPCCallRequest = 5; -} - -/** - * TB Core Data Structures - */ - -message TbSubscriptionProto { - string serviceId = 1; - string sessionId = 2; - int32 subscriptionId = 3; - string entityType = 4; - int64 tenantIdMSB = 5; - int64 tenantIdLSB = 6; - int64 entityIdMSB = 7; - int64 entityIdLSB = 8; -} - -message TbTimeSeriesSubscriptionProto { - TbSubscriptionProto sub = 1; - bool allKeys = 2; - repeated TbSubscriptionKetStateProto keyStates = 3; - int64 startTime = 4; - int64 endTime = 5; -} - -message TbAttributeSubscriptionProto { - TbSubscriptionProto sub = 1; - bool allKeys = 2; - repeated TbSubscriptionKetStateProto keyStates = 3; - string scope = 4; -} - -message TbSubscriptionUpdateProto { - string sessionId = 1; - int32 subscriptionId = 2; - int32 errorCode = 3; - string errorMsg = 4; - repeated TbSubscriptionUpdateValueListProto data = 5; -} - -message TbAttributeUpdateProto { - string entityType = 1; - int64 entityIdMSB = 2; - int64 entityIdLSB = 3; - int64 tenantIdMSB = 4; - int64 tenantIdLSB = 5; - string scope = 6; - repeated TsKvProto data = 7; -} - -message TbTimeSeriesUpdateProto { - string entityType = 1; - int64 entityIdMSB = 2; - int64 entityIdLSB = 3; - int64 tenantIdMSB = 4; - int64 tenantIdLSB = 5; - repeated TsKvProto data = 6; -} - -message TbSubscriptionCloseProto { - string sessionId = 1; - int32 subscriptionId = 2; -} - -message TbSubscriptionKetStateProto { - string key = 1; - int64 ts = 2; -} - -message TbSubscriptionUpdateValueListProto { - string key = 1; - repeated int64 ts = 2; - repeated string value = 3; -} - -/** - * TB Core to TB Core messages - */ - -message DeviceStateServiceMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 deviceIdMSB = 3; - int64 deviceIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} - -message SubscriptionMgrMsgProto { - TbTimeSeriesSubscriptionProto telemetrySub = 1; - TbAttributeSubscriptionProto attributeSub = 2; - TbSubscriptionCloseProto subClose = 3; - TbTimeSeriesUpdateProto tsUpdate = 4; - TbAttributeUpdateProto attrUpdate = 5; -} - -message LocalSubscriptionServiceMsgProto { - TbSubscriptionUpdateProto subUpdate = 1; -} - -message FromDeviceRPCResponseProto { - int64 requestIdMSB = 1; - int64 requestIdLSB = 2; - string response = 3; - int32 error = 4; -} -/** - * Main messages; - */ - -/* Request from Transport Service to ThingsBoard Core Service */ -message TransportApiRequestMsg { - ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; - ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; - GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; - GetTenantRoutingInfoRequestMsg getTenantRoutingInfoRequestMsg = 4; -} - -/* Response from ThingsBoard Core Service to Transport Service */ -message TransportApiResponseMsg { - ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; - GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; - GetTenantRoutingInfoResponseMsg getTenantRoutingInfoResponseMsg = 4; -} - -/* Messages that are handled by ThingsBoard Core Service */ -message ToCoreMsg { - TransportToDeviceActorMsg toDeviceActorMsg = 1; - DeviceStateServiceMsgProto deviceStateServiceMsg = 2; - SubscriptionMgrMsgProto toSubscriptionMgrMsg = 3; - bytes toDeviceActorNotificationMsg = 4; -} - -/* High priority messages with low latency are handled by ThingsBoard Core Service separately */ -message ToCoreNotificationMsg { - LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 1; - FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; - bytes componentLifecycleMsg = 3; -} - -/* Messages that are handled by ThingsBoard RuleEngine Service */ -message ToRuleEngineMsg { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - bytes tbMsg = 3; - repeated string relationTypes = 4; - string failureMessage = 5; -} - -message ToRuleEngineNotificationMsg { - bytes componentLifecycleMsg = 1; - FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; -} - -/* Messages that are handled by ThingsBoard Transport Service */ -message ToTransportMsg { - int64 sessionIdMSB = 1; - int64 sessionIdLSB = 2; - SessionCloseNotificationProto sessionCloseNotification = 3; - GetAttributeResponseMsg getAttributesResponse = 4; - AttributeUpdateNotificationMsg attributeUpdateNotification = 5; - ToDeviceRpcRequestMsg toDeviceRequest = 6; - ToServerRpcResponseMsg toServerResponse = 7; -} diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java index 864e7b2ab9..5be2f76258 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java @@ -28,7 +28,7 @@ import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; * Created by ashvayka on 18.10.18. */ @Slf4j -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") @Component public class CoapTransportContext extends TransportContext { diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index 82678fde5d..9d2cff5c09 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -303,8 +303,6 @@ public class CoapTransportResource extends CoapResource { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) - .setDeviceName(msg.getDeviceInfo().getDeviceName()) - .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index fd4c8e2d95..d725d47a5f 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -20,8 +20,14 @@ import org.eclipse.californium.core.CoapResource; import org.eclipse.californium.core.CoapServer; import org.eclipse.californium.core.network.CoapEndpoint; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.transport.SessionMsgProcessor; +import org.thingsboard.server.common.transport.auth.DeviceAuthService; +import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -30,7 +36,7 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; @Service("CoapTransportService") -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") @Slf4j public class CoapTransportService { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index c2f171e39b..9df658224a 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -61,7 +61,7 @@ import java.util.function.Consumer; * @author Andrew Shvayka */ @RestController -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") @RequestMapping("/api/v1") @Slf4j public class DeviceApiController { @@ -221,8 +221,6 @@ public class DeviceApiController { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) - .setDeviceName(msg.getDeviceInfo().getDeviceName()) - .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java index 28c38bdda6..3d344d8dc5 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java @@ -29,7 +29,7 @@ import javax.annotation.PostConstruct; * Created by ashvayka on 04.10.18. */ @Slf4j -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") @Component public class HttpTransportContext extends TransportContext { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java index b058a1c260..107639f302 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java @@ -15,23 +15,33 @@ */ package org.thingsboard.server.transport.mqtt; +import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.handler.ssl.SslHandler; +import lombok.Data; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import org.thingsboard.server.common.transport.TransportContext; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + /** * Created by ashvayka on 04.10.18. */ @Slf4j +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") @Component -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") public class MqttTransportContext extends TransportContext { @Getter diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 7295cb786b..84c7598d9b 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -44,7 +44,7 @@ import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.service.DefaultTransportService; +import org.thingsboard.server.common.transport.service.AbstractTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; @@ -410,7 +410,13 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private void processDisconnect(ChannelHandlerContext ctx) { ctx.close(); log.info("[{}] Client disconnected!", sessionId); - doDisconnect(); + if (deviceSessionCtx.isConnected()) { + transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); + transportService.deregisterSession(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.onGatewayDisconnect(); + } + } } private MqttConnAckMessage createMqttConnAckMsg(MqttConnectReturnCode returnCode) { @@ -479,17 +485,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void operationComplete(Future future) throws Exception { - doDisconnect(); - } - - private void doDisconnect() { if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); + transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); transportService.deregisterSession(sessionInfo); - if (gatewaySessionHandler != null) { - gatewaySessionHandler.onGatewayDisconnect(); - } - deviceSessionCtx.setDisconnected(); } } @@ -507,10 +505,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement .setDeviceIdLSB(msg.getDeviceInfo().getDeviceIdLSB()) .setTenantIdMSB(msg.getDeviceInfo().getTenantIdMSB()) .setTenantIdLSB(msg.getDeviceInfo().getTenantIdLSB()) - .setDeviceName(msg.getDeviceInfo().getDeviceName()) - .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); - transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.OPEN), null); + transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.OPEN), null); transportService.registerAsyncSession(sessionInfo, this); checkGatewaySession(); ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED)); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java index fb8a0b70be..8f752b5066 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java @@ -36,7 +36,7 @@ import javax.annotation.PreDestroy; * @author Andrew Shvayka */ @Service("MqttTransportService") -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") @Slf4j public class MqttTransportService { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java index b4b5f98238..299f5449d9 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java @@ -44,8 +44,6 @@ public class GatewayDeviceSessionCtx extends MqttDeviceAwareSessionContext imple .setDeviceIdLSB(deviceInfo.getDeviceIdLSB()) .setTenantIdMSB(deviceInfo.getTenantIdMSB()) .setTenantIdLSB(deviceInfo.getTenantIdLSB()) - .setDeviceName(deviceInfo.getDeviceName()) - .setDeviceType(deviceInfo.getDeviceType()) .build(); setDeviceInfo(deviceInfo); } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index de5f82f324..e0a944ed60 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.service.DefaultTransportService; +import org.thingsboard.server.common.transport.service.AbstractTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; @@ -121,7 +121,7 @@ public class GatewaySessionHandler { if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) { SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo(); transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx); - transportService.process(deviceSessionInfo, DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); + transportService.process(deviceSessionInfo, AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null); } @@ -376,7 +376,7 @@ public class GatewaySessionHandler { private void deregisterSession(String deviceName, GatewayDeviceSessionCtx deviceSessionCtx) { transportService.deregisterSession(deviceSessionCtx.getSessionInfo()); - transportService.process(deviceSessionCtx.getSessionInfo(), DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); + transportService.process(deviceSessionCtx.getSessionInfo(), AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); log.debug("[{}] Removed device [{}] from the gateway session", sessionId, deviceName); } diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index b95bc4e54f..16bc802cd3 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -36,10 +36,6 @@ - - org.thingsboard.common - queue - org.thingsboard.common data @@ -103,6 +99,19 @@ org.apache.commons commons-lang3 + + com.google.protobuf + protobuf-java + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + + diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java similarity index 77% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java index bb541fb626..ed1bbf18b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.common.transport; -public interface TbQueueAdmin { +import org.thingsboard.server.common.data.Device; - void createTopicIfNotExists(String topic); +public interface SessionMsgProcessor { + + void onDeviceAdded(Device device); - void destroy(); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java index c49b9c5bd3..ab67beee0b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java @@ -20,9 +20,7 @@ import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -34,16 +32,15 @@ import java.util.concurrent.Executors; */ @Slf4j @Data -@Service -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || '${service.type:null}'=='monolith'") -public abstract class TransportContext { +public class TransportContext { protected final ObjectMapper mapper = new ObjectMapper(); @Autowired private TransportService transportService; + @Autowired - private TbServiceInfoProvider serviceInfoProvider; + private TbNodeIdProvider nodeIdProvider; @Getter private ExecutorService executor; @@ -61,7 +58,7 @@ public abstract class TransportContext { } public String getNodeId() { - return serviceInfoProvider.getServiceId(); + return nodeIdProvider.getNodeId(); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index 2af55c9206..b0772de945 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -15,19 +15,15 @@ */ package org.thingsboard.server.common.transport; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; @@ -39,16 +35,14 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce */ public interface TransportService { - GetTenantRoutingInfoResponseMsg getRoutingInfo(GetTenantRoutingInfoRequestMsg msg); - void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback); void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback); - void process(GetOrCreateDeviceFromGatewayRequestMsg msg, - TransportServiceCallback callback); + void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, + TransportServiceCallback callback); boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback); @@ -68,7 +62,7 @@ public interface TransportService { void process(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback); - void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback); + void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback); void process(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java new file mode 100644 index 0000000000..77c45e1714 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java @@ -0,0 +1,327 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.common.msg.tools.TbRateLimitsException; +import org.thingsboard.server.common.transport.SessionMsgListener; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.*; + +/** + * Created by ashvayka on 17.10.18. + */ +@Slf4j +public abstract class AbstractTransportService implements TransportService { + + @Value("${transport.rate_limits.enabled}") + private boolean rateLimitEnabled; + @Value("${transport.rate_limits.tenant}") + private String perTenantLimitsConf; + @Value("${transport.rate_limits.device}") + private String perDevicesLimitsConf; + @Value("${transport.sessions.inactivity_timeout}") + private long sessionInactivityTimeout; + @Value("${transport.sessions.report_timeout}") + private long sessionReportTimeout; + + protected ScheduledExecutorService schedulerExecutor; + protected ExecutorService transportCallbackExecutor; + + private ConcurrentMap sessions = new ConcurrentHashMap<>(); + + //TODO: Implement cleanup of this maps. + private ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); + private ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); + + @Override + public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { + sessions.putIfAbsent(toId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); + sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); + sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + doProcess(sessionInfo, msg, callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, + TransportServiceCallback callback) { + registerClaimingInfo(sessionInfo, msg, callback); + } + + @Override + public void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { + reportActivityInternal(sessionInfo); + } + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback); + + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback); + + protected abstract void registerClaimingInfo(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback); + + private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { + UUID sessionId = toId(sessionInfo); + SessionMetaData sessionMetaData = sessions.get(sessionId); + if (sessionMetaData != null) { + sessionMetaData.updateLastActivityTime(); + } + return sessionMetaData; + } + + private void checkInactivityAndReportActivity() { + long expTime = System.currentTimeMillis() - sessionInactivityTimeout; + sessions.forEach((uuid, sessionMD) -> { + if (sessionMD.getLastActivityTime() < expTime) { + if (log.isDebugEnabled()) { + log.debug("[{}] Session has expired due to last activity time: {}", toId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); + } + process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); + sessions.remove(uuid); + sessionMD.getListener().onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); + } else { + if (sessionMD.getLastActivityTime() > sessionMD.getLastReportedActivityTime()) { + final long lastActivityTime = sessionMD.getLastActivityTime(); + process(sessionMD.getSessionInfo(), TransportProtos.SubscriptionInfoProto.newBuilder() + .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) + .setRpcSubscription(sessionMD.isSubscribedToRPC()) + .setLastActivityTime(sessionMD.getLastActivityTime()).build(), new TransportServiceCallback() { + @Override + public void onSuccess(Void msg) { + sessionMD.setLastReportedActivityTime(lastActivityTime); + } + + @Override + public void onError(Throwable e) { + log.warn("[{}] Failed to report last activity time", uuid, e); + } + }); + } + } + }); + } + + @Override + public void registerSyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout) { + SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); + sessions.putIfAbsent(toId(sessionInfo), currentSession); + + ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { + listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); + deregisterSession(sessionInfo); + }, timeout, TimeUnit.MILLISECONDS); + + currentSession.setScheduledFuture(executorFuture); + } + + @Override + public void deregisterSession(TransportProtos.SessionInfoProto sessionInfo) { + SessionMetaData currentSession = sessions.get(toId(sessionInfo)); + if (currentSession != null && currentSession.hasScheduledFuture()) { + log.debug("Stopping scheduler to avoid resending response if request has been ack."); + currentSession.getScheduledFuture().cancel(false); + } + sessions.remove(toId(sessionInfo)); + } + + @Override + public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { + if (log.isTraceEnabled()) { + log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); + } + if (!rateLimitEnabled) { + return true; + } + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(perTenantLimitsConf)); + if (!rateLimits.tryConsume()) { + if (callback != null) { + callback.onError(new TbRateLimitsException(EntityType.TENANT)); + } + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Tenant level rate limit detected: {}", toId(sessionInfo), tenantId, msg); + } + return false; + } + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + rateLimits = perDeviceLimits.computeIfAbsent(deviceId, id -> new TbRateLimits(perDevicesLimitsConf)); + if (!rateLimits.tryConsume()) { + if (callback != null) { + callback.onError(new TbRateLimitsException(EntityType.DEVICE)); + } + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Device level rate limit detected: {}", toId(sessionInfo), deviceId, msg); + } + return false; + } + + return true; + } + + protected void processToTransportMsg(TransportProtos.DeviceActorToTransportMsg toSessionMsg) { + UUID sessionId = new UUID(toSessionMsg.getSessionIdMSB(), toSessionMsg.getSessionIdLSB()); + SessionMetaData md = sessions.get(sessionId); + if (md != null) { + SessionMsgListener listener = md.getListener(); + transportCallbackExecutor.submit(() -> { + if (toSessionMsg.hasGetAttributesResponse()) { + listener.onGetAttributesResponse(toSessionMsg.getGetAttributesResponse()); + } + if (toSessionMsg.hasAttributeUpdateNotification()) { + listener.onAttributeUpdate(toSessionMsg.getAttributeUpdateNotification()); + } + if (toSessionMsg.hasSessionCloseNotification()) { + listener.onRemoteSessionCloseCommand(toSessionMsg.getSessionCloseNotification()); + } + if (toSessionMsg.hasToDeviceRequest()) { + listener.onToDeviceRpcRequest(toSessionMsg.getToDeviceRequest()); + } + if (toSessionMsg.hasToServerResponse()) { + listener.onToServerRpcResponse(toSessionMsg.getToServerResponse()); + } + }); + if (md.getSessionType() == TransportProtos.SessionType.SYNC) { + deregisterSession(md.getSessionInfo()); + } + } else { + //TODO: should we notify the device actor about missed session? + log.debug("[{}] Missing session.", sessionId); + } + } + + protected UUID toId(TransportProtos.SessionInfoProto sessionInfo) { + return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); + } + + protected String getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { + return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()).toString(); + } + + public void init() { + if (rateLimitEnabled) { + //Just checking the configuration parameters + new TbRateLimits(perTenantLimitsConf); + new TbRateLimits(perDevicesLimitsConf); + } + this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); + this.transportCallbackExecutor = Executors.newWorkStealingPool(20); + this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); + } + + public void destroy() { + if (rateLimitEnabled) { + perTenantLimits.clear(); + perDeviceLimits.clear(); + } + if (schedulerExecutor != null) { + schedulerExecutor.shutdownNow(); + } + if (transportCallbackExecutor != null) { + transportCallbackExecutor.shutdownNow(); + } + } + + public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { + return TransportProtos.SessionEventMsg.newBuilder() + .setSessionType(TransportProtos.SessionType.ASYNC) + .setEvent(event).build(); + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java deleted file mode 100644 index c820167de1..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ /dev/null @@ -1,600 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.service; - -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.session.SessionMsgType; -import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.msg.tools.TbRateLimitsException; -import org.thingsboard.server.common.transport.SessionMsgListener; -import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.util.JsonUtils; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.AsyncCallbackTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.provider.TbTransportQueueFactory; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Created by ashvayka on 17.10.18. - */ -@Slf4j -@Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport'") -public class DefaultTransportService implements TransportService { - - @Value("${transport.rate_limits.enabled}") - private boolean rateLimitEnabled; - @Value("${transport.rate_limits.tenant}") - private String perTenantLimitsConf; - @Value("${transport.rate_limits.device}") - private String perDevicesLimitsConf; - @Value("${transport.sessions.inactivity_timeout}") - private long sessionInactivityTimeout; - @Value("${transport.sessions.report_timeout}") - private long sessionReportTimeout; - @Value("${transport.client_side_rpc.timeout:60000}") - private long clientSideRpcTimeout; - @Value("${queue.transport.poll_interval}") - private int notificationsPollDuration; - - private final Gson gson = new Gson(); - private final TbTransportQueueFactory queueProvider; - private final TbQueueProducerProvider producerProvider; - private final PartitionService partitionService; - private final TbServiceInfoProvider serviceInfoProvider; - - protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; - protected TbQueueProducer> ruleEngineMsgProducer; - protected TbQueueProducer> tbCoreMsgProducer; - protected TbQueueConsumer> transportNotificationsConsumer; - - protected ScheduledExecutorService schedulerExecutor; - protected ExecutorService transportCallbackExecutor; - - private final ConcurrentMap sessions = new ConcurrentHashMap<>(); - private final Map toServerRpcPendingMap = new ConcurrentHashMap<>(); - //TODO: Implement cleanup of this maps. - private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); - private final ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); - - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); - private volatile boolean stopped = false; - - public DefaultTransportService(TbServiceInfoProvider serviceInfoProvider, TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { - this.serviceInfoProvider = serviceInfoProvider; - this.queueProvider = queueProvider; - this.producerProvider = producerProvider; - this.partitionService = partitionService; - } - - @PostConstruct - public void init() { - if (rateLimitEnabled) { - //Just checking the configuration parameters - new TbRateLimits(perTenantLimitsConf); - new TbRateLimits(perDevicesLimitsConf); - } - this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); - this.transportCallbackExecutor = Executors.newWorkStealingPool(20); - this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); - transportApiRequestTemplate = queueProvider.createTransportApiRequestTemplate(); - ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); - tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); - transportNotificationsConsumer = queueProvider.createTransportNotificationsConsumer(); - TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceInfoProvider.getServiceId()); - transportNotificationsConsumer.subscribe(Collections.singleton(tpi)); - transportApiRequestTemplate.init(); - mainConsumerExecutor.execute(() -> { - while (!stopped) { - try { - List> records = transportNotificationsConsumer.poll(notificationsPollDuration); - if (records.size() == 0) { - continue; - } - records.forEach(record -> { - try { - processToTransportMsg(record.getValue()); - } catch (Throwable e) { - log.warn("Failed to process the notification.", e); - } - }); - transportNotificationsConsumer.commit(); - } catch (Exception e) { - if (!stopped) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(notificationsPollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - } - }); - } - - @PreDestroy - public void destroy() { - if (rateLimitEnabled) { - perTenantLimits.clear(); - perDeviceLimits.clear(); - } - stopped = true; - - if (transportNotificationsConsumer != null) { - transportNotificationsConsumer.unsubscribe(); - } - if (schedulerExecutor != null) { - schedulerExecutor.shutdownNow(); - } - if (transportCallbackExecutor != null) { - transportCallbackExecutor.shutdownNow(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - if (transportApiRequestTemplate != null) { - transportApiRequestTemplate.stop(); - } - } - - @Override - public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { - sessions.putIfAbsent(toSessionId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); - } - - @Override - public TransportProtos.GetTenantRoutingInfoResponseMsg getRoutingInfo(TransportProtos.GetTenantRoutingInfoRequestMsg msg) { - TbProtoQueueMsg protoMsg = - new TbProtoQueueMsg<>(UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder().setGetTenantRoutingInfoRequestMsg(msg).build()); - try { - TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); - return response.getValue().getGetTenantRoutingInfoResponseMsg(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - } - - @Override - public void process(TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(TransportProtos.ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { - if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); - } - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscriptionInfo(msg).build(), callback); - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSessionEvent(msg).build(), callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - MsgPackCallback packCallback = new MsgPackCallback(msg.getTsKvListCount(), callback); - for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) { - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("deviceName", sessionInfo.getDeviceName()); - metaData.putValue("deviceType", sessionInfo.getDeviceType()); - metaData.putValue("ts", tsKv.getTs() + ""); - JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); - TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, gson.toJson(json)); - sendToRuleEngine(tenantId, tbMsg, packCallback); - } - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("deviceName", sessionInfo.getDeviceName()); - metaData.putValue("deviceType", sessionInfo.getDeviceType()); - TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, gson.toJson(json)); - sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setGetAttributes(msg).build(), callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); - sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToAttributes(msg).build(), callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); - sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToRPC(msg).build(), callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToDeviceRPCCallResponse(msg).build(), callback); - } - } - - private void processTimeout(String requestId) { - RpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); - if (data != null) { - SessionMetaData md = sessions.get(data.getSessionId()); - if (md != null) { - SessionMsgListener listener = md.getListener(); - transportCallbackExecutor.submit(() -> { - TransportProtos.ToServerRpcResponseMsg responseMsg = - TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(data.getRequestId()) - .setError("timeout").build(); - listener.onToServerRpcResponse(responseMsg); - }); - if (md.getSessionType() == TransportProtos.SessionType.SYNC) { - deregisterSession(md.getSessionInfo()); - } - } else { - log.debug("[{}] Missing session.", data.getSessionId()); - } - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - UUID sessionId = toSessionId(sessionInfo); - TenantId tenantId = getTenantId(sessionInfo); - DeviceId deviceId = getDeviceId(sessionInfo); - JsonObject json = new JsonObject(); - json.addProperty("method", msg.getMethodName()); - json.add("params", JsonUtils.parse(msg.getParams())); - - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("deviceName", sessionInfo.getDeviceName()); - metaData.putValue("deviceType", sessionInfo.getDeviceType()); - metaData.putValue("requestId", Integer.toString(msg.getRequestId())); - metaData.putValue("serviceId", serviceInfoProvider.getServiceId()); - metaData.putValue("sessionId", sessionId.toString()); - TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json)); - sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); - - String requestId = sessionId + "-" + msg.getRequestId(); - toServerRpcPendingMap.put(requestId, new RpcRequestMetadata(sessionId, msg.getRequestId())); - schedulerExecutor.schedule(() -> processTimeout(requestId), clientSideRpcTimeout, TimeUnit.MILLISECONDS); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setClaimDevice(msg).build(), callback); - } - } - - @Override - public void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { - reportActivityInternal(sessionInfo); - } - - private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { - UUID sessionId = toSessionId(sessionInfo); - SessionMetaData sessionMetaData = sessions.get(sessionId); - if (sessionMetaData != null) { - sessionMetaData.updateLastActivityTime(); - } - return sessionMetaData; - } - - private void checkInactivityAndReportActivity() { - long expTime = System.currentTimeMillis() - sessionInactivityTimeout; - sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getLastActivityTime() < expTime) { - if (log.isDebugEnabled()) { - log.debug("[{}] Session has expired due to last activity time: {}", toSessionId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); - } - process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); - sessions.remove(uuid); - sessionMD.getListener().onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); - } else { - if (sessionMD.getLastActivityTime() > sessionMD.getLastReportedActivityTime()) { - final long lastActivityTime = sessionMD.getLastActivityTime(); - process(sessionMD.getSessionInfo(), TransportProtos.SubscriptionInfoProto.newBuilder() - .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) - .setRpcSubscription(sessionMD.isSubscribedToRPC()) - .setLastActivityTime(sessionMD.getLastActivityTime()).build(), new TransportServiceCallback() { - @Override - public void onSuccess(Void msg) { - sessionMD.setLastReportedActivityTime(lastActivityTime); - } - - @Override - public void onError(Throwable e) { - log.warn("[{}] Failed to report last activity time", uuid, e); - } - }); - } - } - }); - } - - @Override - public void registerSyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout) { - SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); - sessions.putIfAbsent(toSessionId(sessionInfo), currentSession); - - ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { - listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); - deregisterSession(sessionInfo); - }, timeout, TimeUnit.MILLISECONDS); - - currentSession.setScheduledFuture(executorFuture); - } - - @Override - public void deregisterSession(TransportProtos.SessionInfoProto sessionInfo) { - SessionMetaData currentSession = sessions.get(toSessionId(sessionInfo)); - if (currentSession != null && currentSession.hasScheduledFuture()) { - log.debug("Stopping scheduler to avoid resending response if request has been ack."); - currentSession.getScheduledFuture().cancel(false); - } - sessions.remove(toSessionId(sessionInfo)); - } - - @Override - public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { - if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); - } - if (!rateLimitEnabled) { - return true; - } - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(perTenantLimitsConf)); - if (!rateLimits.tryConsume()) { - if (callback != null) { - callback.onError(new TbRateLimitsException(EntityType.TENANT)); - } - if (log.isTraceEnabled()) { - log.trace("[{}][{}] Tenant level rate limit detected: {}", toSessionId(sessionInfo), tenantId, msg); - } - return false; - } - DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - rateLimits = perDeviceLimits.computeIfAbsent(deviceId, id -> new TbRateLimits(perDevicesLimitsConf)); - if (!rateLimits.tryConsume()) { - if (callback != null) { - callback.onError(new TbRateLimitsException(EntityType.DEVICE)); - } - if (log.isTraceEnabled()) { - log.trace("[{}][{}] Device level rate limit detected: {}", toSessionId(sessionInfo), deviceId, msg); - } - return false; - } - - return true; - } - - protected void processToTransportMsg(TransportProtos.ToTransportMsg toSessionMsg) { - UUID sessionId = new UUID(toSessionMsg.getSessionIdMSB(), toSessionMsg.getSessionIdLSB()); - SessionMetaData md = sessions.get(sessionId); - if (md != null) { - SessionMsgListener listener = md.getListener(); - transportCallbackExecutor.submit(() -> { - if (toSessionMsg.hasGetAttributesResponse()) { - listener.onGetAttributesResponse(toSessionMsg.getGetAttributesResponse()); - } - if (toSessionMsg.hasAttributeUpdateNotification()) { - listener.onAttributeUpdate(toSessionMsg.getAttributeUpdateNotification()); - } - if (toSessionMsg.hasSessionCloseNotification()) { - listener.onRemoteSessionCloseCommand(toSessionMsg.getSessionCloseNotification()); - } - if (toSessionMsg.hasToDeviceRequest()) { - listener.onToDeviceRpcRequest(toSessionMsg.getToDeviceRequest()); - } - if (toSessionMsg.hasToServerResponse()) { - String requestId = sessionId + "-" + toSessionMsg.getToServerResponse().getRequestId(); - toServerRpcPendingMap.remove(requestId); - listener.onToServerRpcResponse(toSessionMsg.getToServerResponse()); - } - }); - if (md.getSessionType() == TransportProtos.SessionType.SYNC) { - deregisterSession(md.getSessionInfo()); - } - } else { - //TODO: should we notify the device actor about missed session? - log.debug("[{}] Missing session.", sessionId); - } - } - - protected UUID toSessionId(TransportProtos.SessionInfoProto sessionInfo) { - return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); - } - - protected UUID getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { - return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()); - } - - protected TenantId getTenantId(TransportProtos.SessionInfoProto sessionInfo) { - return new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - } - - protected DeviceId getDeviceId(TransportProtos.SessionInfoProto sessionInfo) { - return new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - } - - public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { - return TransportProtos.SessionEventMsg.newBuilder() - .setSessionType(TransportProtos.SessionType.ASYNC) - .setEvent(event).build(); - } - - protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); - tbCoreMsgProducer.send(tpi, - new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), - ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? - new TransportTbQueueCallback(callback) : null); - } - - protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); - ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); - ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); - } - - private class TransportTbQueueCallback implements TbQueueCallback { - private final TransportServiceCallback callback; - - private TransportTbQueueCallback(TransportServiceCallback callback) { - this.callback = callback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null)); - } - - @Override - public void onFailure(Throwable t) { - DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onError(t)); - } - } - - private class MsgPackCallback implements TbQueueCallback { - private final AtomicInteger msgCount; - private final TransportServiceCallback callback; - - public MsgPackCallback(Integer msgCount, TransportServiceCallback callback) { - this.msgCount = new AtomicInteger(msgCount); - this.callback = callback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (msgCount.decrementAndGet() <= 0) { - DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null)); - } - } - - @Override - public void onFailure(Throwable t) { - callback.onError(t); - } - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java new file mode 100644 index 0000000000..17031791d5 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java @@ -0,0 +1,339 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.kafka.AsyncCallbackTemplate; +import org.thingsboard.server.kafka.TBKafkaAdmin; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaRequestTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.time.Duration; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Created by ashvayka on 05.10.18. + */ +@ConditionalOnExpression("'${transport.type:null}'=='null'") +@Service +@Slf4j +public class RemoteTransportService extends AbstractTransportService { + + @Value("${kafka.rule_engine.topic}") + private String ruleEngineTopic; + @Value("${kafka.notifications.topic}") + private String notificationsTopic; + @Value("${kafka.notifications.poll_interval}") + private int notificationsPollDuration; + @Value("${kafka.notifications.auto_commit_interval}") + private int notificationsAutoCommitInterval; + @Value("${kafka.transport_api.requests_topic}") + private String transportApiRequestsTopic; + @Value("${kafka.transport_api.responses_topic}") + private String transportApiResponsesTopic; + @Value("${kafka.transport_api.max_pending_requests}") + private long maxPendingRequests; + @Value("${kafka.transport_api.max_requests_timeout}") + private long maxRequestsTimeout; + @Value("${kafka.transport_api.response_poll_interval}") + private int responsePollDuration; + @Value("${kafka.transport_api.response_auto_commit_interval}") + private int autoCommitInterval; + + @Autowired + private TbKafkaSettings kafkaSettings; + //We use this to get the node id. We should replace this with a component that provides the node id. + @Autowired + private TbNodeIdProvider nodeIdProvider; + + private TbKafkaRequestTemplate transportApiTemplate; + private TBKafkaProducerTemplate ruleEngineProducer; + private TBKafkaConsumerTemplate mainConsumer; + + private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("remote-transport-consumer")); + + private volatile boolean stopped = false; + + @PostConstruct + public void init() { + super.init(); + + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-api-request-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportApiRequestsTopic); + requestBuilder.encoder(new TransportApiRequestEncoder()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(transportApiResponsesTopic + "." + nodeIdProvider.getNodeId()); + responseBuilder.clientId("transport-api-client-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("transport-api-client"); + responseBuilder.autoCommit(true); + responseBuilder.autoCommitIntervalMs(autoCommitInterval); + responseBuilder.decoder(new TransportApiResponseDecoder()); + + TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder + builder = TbKafkaRequestTemplate.builder(); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(maxPendingRequests); + builder.maxRequestTimeout(maxRequestsTimeout); + builder.pollInterval(responsePollDuration); + transportApiTemplate = builder.build(); + transportApiTemplate.init(); + + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder ruleEngineProducerBuilder = TBKafkaProducerTemplate.builder(); + ruleEngineProducerBuilder.settings(kafkaSettings); + ruleEngineProducerBuilder.clientId("producer-rule-engine-request-" + nodeIdProvider.getNodeId()); + ruleEngineProducerBuilder.defaultTopic(ruleEngineTopic); + ruleEngineProducerBuilder.encoder(new ToRuleEngineMsgEncoder()); + ruleEngineProducer = ruleEngineProducerBuilder.build(); + ruleEngineProducer.init(); + + String notificationsTopicName = notificationsTopic + "." + nodeIdProvider.getNodeId(); + + try { + TBKafkaAdmin admin = new TBKafkaAdmin(kafkaSettings); + CreateTopicsResult result = admin.createTopic(new NewTopic(notificationsTopicName, 1, (short) 1)); + result.all().get(); + } catch (Exception e) { + log.trace("Failed to create topic: {}", e.getMessage(), e); + } + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder mainConsumerBuilder = TBKafkaConsumerTemplate.builder(); + mainConsumerBuilder.settings(kafkaSettings); + mainConsumerBuilder.topic(notificationsTopicName); + mainConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); + mainConsumerBuilder.groupId("transport"); + mainConsumerBuilder.autoCommit(true); + mainConsumerBuilder.autoCommitIntervalMs(notificationsAutoCommitInterval); + mainConsumerBuilder.decoder(new ToTransportMsgResponseDecoder()); + mainConsumer = mainConsumerBuilder.build(); + mainConsumer.subscribe(); + + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + ConsumerRecords records = mainConsumer.poll(Duration.ofMillis(notificationsPollDuration)); + records.forEach(record -> { + try { + ToTransportMsg toTransportMsg = mainConsumer.decode(record); + if (toTransportMsg.hasToDeviceSessionMsg()) { + processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); + } + } catch (Throwable e) { + log.warn("Failed to process the notification.", e); + } + }); + } catch (Exception e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(notificationsPollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + @PreDestroy + public void destroy() { + super.destroy(); + stopped = true; + if (transportApiTemplate != null) { + transportApiTemplate.stop(); + } + if (mainConsumer != null) { + mainConsumer.unsubscribe(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + } + + @Override + public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getToken(), + TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), + response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getHash(), + TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), + response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getDeviceName(), + TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), + response -> callback.onSuccess(response.getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback) { + if (log.isTraceEnabled()) { + log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); + } + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscriptionInfo(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSessionEvent(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setPostTelemetry(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setPostAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setGetAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToRPC(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToDeviceRPCCallResponse(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToServerRPCCallRequest(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + @Override + protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setClaimDevice(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + + private void send(SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { + ruleEngineProducer.send(getRoutingKey(sessionInfo), toRuleEngineMsg, (metadata, exception) -> { + if (callback != null) { + if (exception == null) { + this.transportCallbackExecutor.submit(() -> { + callback.onSuccess(null); + }); + } else { + this.transportCallbackExecutor.submit(() -> { + callback.onError(exception); + }); + } + } + }); + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java index fc1445539b..42a65440a0 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.kafka.TbKafkaEncoder; +import org.thingsboard.server.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java index cc154f87b2..c07e406d2e 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java @@ -15,9 +15,8 @@ */ package org.thingsboard.server.common.transport.service; -import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.kafka.TbKafkaDecoder; +import org.thingsboard.server.kafka.TbKafkaDecoder; import java.io.IOException; @@ -25,9 +24,8 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class ToTransportMsgResponseDecoder implements TbKafkaDecoder { - @Override - public ToTransportMsg decode(TbQueueMsg msg) throws IOException { - return ToTransportMsg.parseFrom(msg.getData()); + public ToTransportMsg decode(byte[] data) throws IOException { + return ToTransportMsg.parseFrom(data); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java index 3971fdbf83..6c4b362f03 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.queue.kafka.TbKafkaEncoder; +import org.thingsboard.server.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java index 5fdc58c068..a62e696500 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java @@ -15,9 +15,8 @@ */ package org.thingsboard.server.common.transport.service; -import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.kafka.TbKafkaDecoder; +import org.thingsboard.server.kafka.TbKafkaDecoder; import java.io.IOException; @@ -25,9 +24,8 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class TransportApiResponseDecoder implements TbKafkaDecoder { - @Override - public TransportApiResponseMsg decode(TbQueueMsg msg) throws IOException { - return TransportApiResponseMsg.parseFrom(msg.getData()); + public TransportApiResponseMsg decode(byte[] data) throws IOException { + return TransportApiResponseMsg.parseFrom(data); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java deleted file mode 100644 index f2534a64c9..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.service; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; -import org.thingsboard.server.queue.discovery.TenantRoutingInfo; -import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; - -@Slf4j -@Service -@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") -public class TransportTenantRoutingInfoService implements TenantRoutingInfoService { - - private TransportService transportService; - - @Lazy - @Autowired - public void setTransportService(TransportService transportService) { - this.transportService = transportService; - } - - @Override - public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { - GetTenantRoutingInfoRequestMsg msg = GetTenantRoutingInfoRequestMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .build(); - GetTenantRoutingInfoResponseMsg routingInfo = transportService.getRoutingInfo(msg); - return new TenantRoutingInfo(tenantId, routingInfo.getIsolatedTbCore(), routingInfo.getIsolatedTbRuleEngine()); - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java index c312b3a4d9..7c590ec925 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java @@ -35,7 +35,6 @@ public abstract class DeviceAwareSessionContext implements SessionContext { private volatile DeviceId deviceId; @Getter private volatile DeviceInfoProto deviceInfo; - private volatile boolean connected; public DeviceId getDeviceId() { return deviceId; @@ -43,15 +42,10 @@ public abstract class DeviceAwareSessionContext implements SessionContext { public void setDeviceInfo(DeviceInfoProto deviceInfo) { this.deviceInfo = deviceInfo; - this.connected = true; this.deviceId = new DeviceId(new UUID(deviceInfo.getDeviceIdMSB(), deviceInfo.getDeviceIdLSB())); } public boolean isConnected() { - return connected; - } - - public void setDisconnected() { - this.connected = false; + return deviceInfo != null; } } diff --git a/common/transport/transport-api/src/main/proto/transport.proto b/common/transport/transport-api/src/main/proto/transport.proto new file mode 100644 index 0000000000..e8b513574a --- /dev/null +++ b/common/transport/transport-api/src/main/proto/transport.proto @@ -0,0 +1,244 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; +package transport; + +option java_package = "org.thingsboard.server.gen.transport"; +option java_outer_classname = "TransportProtos"; + +/** + * Data Structures; + */ +message SessionInfoProto { + string nodeId = 1; + int64 sessionIdMSB = 2; + int64 sessionIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + int64 deviceIdMSB = 6; + int64 deviceIdLSB = 7; +} + +enum SessionEvent { + OPEN = 0; + CLOSED = 1; +} + +enum SessionType { + SYNC = 0; + ASYNC = 1; +} + +enum KeyValueType { + BOOLEAN_V = 0; + LONG_V = 1; + DOUBLE_V = 2; + STRING_V = 3; + JSON_V = 4; +} + +message KeyValueProto { + string key = 1; + KeyValueType type = 2; + bool bool_v = 3; + int64 long_v = 4; + double double_v = 5; + string string_v = 6; + string json_v = 7; +} + +message TsKvProto { + int64 ts = 1; + KeyValueProto kv = 2; +} + +message TsKvListProto { + int64 ts = 1; + repeated KeyValueProto kv = 2; +} + +message DeviceInfoProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + string deviceName = 5; + string deviceType = 6; + string additionalInfo = 7; +} + +/** + * Messages that use Data Structures; + */ +message SessionEventMsg { + SessionType sessionType = 1; + SessionEvent event = 2; +} + +message PostTelemetryMsg { + repeated TsKvListProto tsKvList = 1; +} + +message PostAttributeMsg { + repeated KeyValueProto kv = 1; +} + +message GetAttributeRequestMsg { + int32 requestId = 1; + repeated string clientAttributeNames = 2; + repeated string sharedAttributeNames = 3; +} + +message GetAttributeResponseMsg { + int32 requestId = 1; + repeated TsKvProto clientAttributeList = 2; + repeated TsKvProto sharedAttributeList = 3; + repeated string deletedAttributeKeys = 4; + string error = 5; +} + +message AttributeUpdateNotificationMsg { + repeated TsKvProto sharedUpdated = 1; + repeated string sharedDeleted = 2; +} + +message ValidateDeviceTokenRequestMsg { + string token = 1; +} + +message ValidateDeviceX509CertRequestMsg { + string hash = 1; +} + +message ValidateDeviceCredentialsResponseMsg { + DeviceInfoProto deviceInfo = 1; + string credentialsBody = 2; +} + +message GetOrCreateDeviceFromGatewayRequestMsg { + int64 gatewayIdMSB = 1; + int64 gatewayIdLSB = 2; + string deviceName = 3; + string deviceType = 4; +} + +message GetOrCreateDeviceFromGatewayResponseMsg { + DeviceInfoProto deviceInfo = 1; +} + +message SessionCloseNotificationProto { + string message = 1; +} + +message SubscribeToAttributeUpdatesMsg { + bool unsubscribe = 1; +} + +message SubscribeToRPCMsg { + bool unsubscribe = 1; +} + +message ToDeviceRpcRequestMsg { + int32 requestId = 1; + string methodName = 2; + string params = 3; +} + +message ToDeviceRpcResponseMsg { + int32 requestId = 1; + string payload = 2; +} + +message ToServerRpcRequestMsg { + int32 requestId = 1; + string methodName = 2; + string params = 3; +} + +message ToServerRpcResponseMsg { + int32 requestId = 1; + string payload = 2; + string error = 3; +} + +message ClaimDeviceMsg { + int64 deviceIdMSB = 1; + int64 deviceIdLSB = 2; + string secretKey = 3; + int64 durationMs = 4; +} + +//Used to report session state to tb-node and persist this state in the cache on the tb-node level. +message SubscriptionInfoProto { + int64 lastActivityTime = 1; + bool attributeSubscription = 2; + bool rpcSubscription = 3; +} + +message SessionSubscriptionInfoProto { + SessionInfoProto sessionInfo = 1; + SubscriptionInfoProto subscriptionInfo = 2; +} + +message DeviceSessionsCacheEntry { + repeated SessionSubscriptionInfoProto sessions = 1; +} + +message TransportToDeviceActorMsg { + SessionInfoProto sessionInfo = 1; + SessionEventMsg sessionEvent = 2; + PostTelemetryMsg postTelemetry = 3; + PostAttributeMsg postAttributes = 4; + GetAttributeRequestMsg getAttributes = 5; + SubscribeToAttributeUpdatesMsg subscribeToAttributes = 6; + SubscribeToRPCMsg subscribeToRPC = 7; + ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 8; + ToServerRpcRequestMsg toServerRPCCallRequest = 9; + SubscriptionInfoProto subscriptionInfo = 10; + ClaimDeviceMsg claimDevice = 11; +} + +message DeviceActorToTransportMsg { + int64 sessionIdMSB = 1; + int64 sessionIdLSB = 2; + SessionCloseNotificationProto sessionCloseNotification = 3; + GetAttributeResponseMsg getAttributesResponse = 4; + AttributeUpdateNotificationMsg attributeUpdateNotification = 5; + ToDeviceRpcRequestMsg toDeviceRequest = 6; + ToServerRpcResponseMsg toServerResponse = 7; +} + +/** + * Main messages; + */ +message ToRuleEngineMsg { + TransportToDeviceActorMsg toDeviceActorMsg = 1; +} + +message ToTransportMsg { + DeviceActorToTransportMsg toDeviceSessionMsg = 1; +} + +message TransportApiRequestMsg { + ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; + ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; + GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; +} + +message TransportApiResponseMsg { + ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; + GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 49926ad5c1..8148fccf79 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -29,7 +29,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 7080722056..5867e505b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -23,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java index b5bda06a47..386220df55 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java @@ -19,8 +19,8 @@ import java.util.UUID; public interface BaseEntity extends ToData { - UUID getUuid(); + UUID getId(); - void setUuid(UUID id); + void setId(UUID id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java index 44fd70f977..d75262e311 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java @@ -35,15 +35,14 @@ public abstract class BaseSqlEntity implements BaseEntity { protected String id; @Override - public UUID getUuid() { + public UUID getId() { if (id == null) { return null; } return UUIDConverter.fromString(id); } - @Override - public void setUuid(UUID id) { + public void setId(UUID id) { this.id = UUIDConverter.fromTimeUUID(id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 9f72567838..96ce14c459 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -32,9 +32,6 @@ public class ModelConstants { public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID); public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); - // this is the difference between midnight October 15, 1582 UTC and midnight January 1, 1970 UTC as 100 nanosecond units - public static final long EPOCH_DIFF = 122192928000000000L; - /** * Generic constants. */ @@ -115,8 +112,6 @@ public class ModelConstants { public static final String TENANT_TITLE_PROPERTY = TITLE_PROPERTY; public static final String TENANT_REGION_PROPERTY = "region"; public static final String TENANT_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY; - public static final String TENANT_ISOLATED_TB_CORE = "isolated_tb_core"; - public static final String TENANT_ISOLATED_TB_RULE_ENGINE = "isolated_tb_rule_engine"; public static final String TENANT_BY_REGION_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "tenant_by_region_and_search_text"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java index 22d8b1c212..5e822a518c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAlarmEntity.java @@ -26,7 +26,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -110,7 +110,7 @@ public abstract class AbstractAlarmEntity extends BaseSqlEntity public AbstractAlarmEntity(Alarm alarm) { if (alarm.getId() != null) { - this.setUuid(alarm.getId().getId()); + this.setId(alarm.getId().getId()); } if (alarm.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(alarm.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java index fac8d68f62..4ab6144649 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java @@ -75,7 +75,7 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity public AbstractAssetEntity(Asset asset) { if (asset.getId() != null) { - this.setUuid(asset.getId().getId()); + this.setId(asset.getId().getId()); } if (asset.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(asset.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java index 042fcec950..6a973335e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java @@ -68,7 +68,7 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti public AbstractDeviceEntity(Device device) { if (device.getId() != null) { - this.setUuid(device.getId().getId()); + this.setId(device.getId().getId()); } if (device.getTenantId() != null) { this.tenantId = toString(device.getTenantId().getId()); @@ -104,8 +104,8 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti } protected Device toDevice() { - Device device = new Device(new DeviceId(getUuid())); - device.setCreatedTime(UUIDs.unixTimestamp(getUuid())); + Device device = new Device(new DeviceId(getId())); + device.setCreatedTime(UUIDs.unixTimestamp(getId())); if (tenantId != null) { device.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java index 3d140039ce..c5f0fd36f2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java @@ -94,7 +94,7 @@ public abstract class AbstractEntityViewEntity extends Bas public AbstractEntityViewEntity(EntityView entityView) { if (entityView.getId() != null) { - this.setUuid(entityView.getId().getId()); + this.setId(entityView.getId().getId()); } if (entityView.getEntityId() != null) { this.entityId = toString(entityView.getEntityId().getId()); @@ -145,8 +145,8 @@ public abstract class AbstractEntityViewEntity extends Bas } protected EntityView toEntityView() { - EntityView entityView = new EntityView(new EntityViewId(getUuid())); - entityView.setCreatedTime(UUIDs.unixTimestamp(getUuid())); + EntityView entityView = new EntityView(new EntityViewId(getId())); + entityView.setCreatedTime(UUIDs.unixTimestamp(getId())); if (entityId != null) { entityView.setEntityId(EntityIdFactory.getByTypeAndId(entityType.name(), toUUID(entityId).toString())); @@ -169,5 +169,4 @@ public abstract class AbstractEntityViewEntity extends Bas entityView.setAdditionalInfo(additionalInfo); return entityView; } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java index 1b1a27a04f..21c411d3e2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java @@ -56,7 +56,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl public AdminSettingsEntity(AdminSettings adminSettings) { if (adminSettings.getId() != null) { - this.setUuid(adminSettings.getId().getId()); + this.setId(adminSettings.getId().getId()); } this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java index d05e3cdde1..fa9490747a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java @@ -19,7 +19,6 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.alarm.Alarm; - import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Entity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java index e2573ea0d1..4de738ac73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java @@ -103,7 +103,7 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit public AuditLogEntity(AuditLog auditLog) { if (auditLog.getId() != null) { - this.setUuid(auditLog.getId().getId()); + this.setId(auditLog.getId().getId()); } if (auditLog.getTenantId() != null) { this.tenantId = toString(auditLog.getTenantId().getId()); @@ -128,8 +128,8 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit @Override public AuditLog toData() { - AuditLog auditLog = new AuditLog(new AuditLogId(this.getUuid())); - auditLog.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + AuditLog auditLog = new AuditLog(new AuditLogId(getId())); + auditLog.setCreatedTime(UUIDs.unixTimestamp(getId())); if (tenantId != null) { auditLog.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java index d5b287fe9b..26946cf207 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java @@ -34,6 +34,7 @@ import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Table; +import javax.persistence.UniqueConstraint; @Data @EqualsAndHashCode(callSuper = true) @@ -71,7 +72,7 @@ public class ComponentDescriptorEntity extends BaseSqlEntity implements Sea public CustomerEntity(Customer customer) { if (customer.getId() != null) { - this.setUuid(customer.getId().getId()); + this.setId(customer.getId().getId()); } this.tenantId = UUIDConverter.fromTimeUUID(customer.getTenantId().getId()); this.title = customer.getTitle(); @@ -111,8 +111,8 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea @Override public Customer toData() { - Customer customer = new Customer(new CustomerId(this.getUuid())); - customer.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + Customer customer = new Customer(new CustomerId(getId())); + customer.setCreatedTime(UUIDs.unixTimestamp(getId())); customer.setTenantId(new TenantId(UUIDConverter.fromString(tenantId))); customer.setTitle(title); customer.setCountry(country); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java index 07cec4d12f..b606c35d4e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java @@ -75,7 +75,7 @@ public final class DashboardEntity extends BaseSqlEntity implements S public DashboardEntity(Dashboard dashboard) { if (dashboard.getId() != null) { - this.setUuid(dashboard.getId().getId()); + this.setId(dashboard.getId().getId()); } if (dashboard.getTenantId() != null) { this.tenantId = toString(dashboard.getTenantId().getId()); @@ -103,8 +103,8 @@ public final class DashboardEntity extends BaseSqlEntity implements S @Override public Dashboard toData() { - Dashboard dashboard = new Dashboard(new DashboardId(this.getUuid())); - dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + Dashboard dashboard = new Dashboard(new DashboardId(this.getId())); + dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getId())); if (tenantId != null) { dashboard.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java index f26cf21934..b1ecb2572b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java @@ -66,7 +66,7 @@ public class DashboardInfoEntity extends BaseSqlEntity implements public DashboardInfoEntity(DashboardInfo dashboardInfo) { if (dashboardInfo.getId() != null) { - this.setUuid(dashboardInfo.getId().getId()); + this.setId(dashboardInfo.getId().getId()); } if (dashboardInfo.getTenantId() != null) { this.tenantId = toString(dashboardInfo.getTenantId().getId()); @@ -97,8 +97,8 @@ public class DashboardInfoEntity extends BaseSqlEntity implements @Override public DashboardInfo toData() { - DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(this.getUuid())); - dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(getId())); + dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(getId())); if (tenantId != null) { dashboardInfo.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java index ba9d7ce637..1d47c7d63c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java @@ -57,7 +57,7 @@ public final class DeviceCredentialsEntity extends BaseSqlEntity implements BaseEntity implements BaseEntity implements BaseEntity implements SearchT public RuleChainEntity(RuleChain ruleChain) { if (ruleChain.getId() != null) { - this.setUuid(ruleChain.getUuidId()); + this.setId(ruleChain.getUuidId()); } this.tenantId = toString(DaoUtil.getId(ruleChain.getTenantId())); this.name = ruleChain.getName(); @@ -100,8 +100,8 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT @Override public RuleChain toData() { - RuleChain ruleChain = new RuleChain(new RuleChainId(this.getUuid())); - ruleChain.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + RuleChain ruleChain = new RuleChain(new RuleChainId(getId())); + ruleChain.setCreatedTime(UUIDs.unixTimestamp(getId())); ruleChain.setTenantId(new TenantId(toUUID(tenantId))); ruleChain.setName(name); if (firstRuleNodeId != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java index 1a7866418a..f36edc260b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java @@ -69,7 +69,7 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex public RuleNodeEntity(RuleNode ruleNode) { if (ruleNode.getId() != null) { - this.setUuid(ruleNode.getUuidId()); + this.setId(ruleNode.getUuidId()); } if (ruleNode.getRuleChainId() != null) { this.ruleChainId = toString(DaoUtil.getId(ruleNode.getRuleChainId())); @@ -94,8 +94,8 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex @Override public RuleNode toData() { - RuleNode ruleNode = new RuleNode(new RuleNodeId(this.getUuid())); - ruleNode.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + RuleNode ruleNode = new RuleNode(new RuleNodeId(getId())); + ruleNode.setCreatedTime(UUIDs.unixTimestamp(getId())); if (ruleChainId != null) { ruleNode.setRuleChainId(new RuleChainId(toUUID(ruleChainId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java index 1da71224f0..239d3698b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java @@ -72,12 +72,6 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Column(name = ModelConstants.EMAIL_PROPERTY) private String email; - @Column(name = ModelConstants.TENANT_ISOLATED_TB_CORE) - private boolean isolatedTbCore; - - @Column(name = ModelConstants.TENANT_ISOLATED_TB_RULE_ENGINE) - private boolean isolatedTbRuleEngine; - @Type(type = "json") @Column(name = ModelConstants.TENANT_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; @@ -88,7 +82,7 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT public TenantEntity(Tenant tenant) { if (tenant.getId() != null) { - this.setUuid(tenant.getId().getId()); + this.setId(tenant.getId().getId()); } this.title = tenant.getTitle(); this.region = tenant.getRegion(); @@ -101,8 +95,6 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT this.phone = tenant.getPhone(); this.email = tenant.getEmail(); this.additionalInfo = tenant.getAdditionalInfo(); - this.isolatedTbCore = tenant.isIsolatedTbCore(); - this.isolatedTbRuleEngine = tenant.isIsolatedTbRuleEngine(); } @Override @@ -121,8 +113,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Override public Tenant toData() { - Tenant tenant = new Tenant(new TenantId(this.getUuid())); - tenant.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + Tenant tenant = new Tenant(new TenantId(getId())); + tenant.setCreatedTime(UUIDs.unixTimestamp(getId())); tenant.setTitle(title); tenant.setRegion(region); tenant.setCountry(country); @@ -134,8 +126,6 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT tenant.setPhone(phone); tenant.setEmail(email); tenant.setAdditionalInfo(additionalInfo); - tenant.setIsolatedTbCore(isolatedTbCore); - tenant.setIsolatedTbRuleEngine(isolatedTbRuleEngine); return tenant; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java index 6957f87961..70593ca610 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java @@ -56,7 +56,7 @@ public final class UserCredentialsEntity extends BaseSqlEntity public UserCredentialsEntity(UserCredentials userCredentials) { if (userCredentials.getId() != null) { - this.setUuid(userCredentials.getId().getId()); + this.setId(userCredentials.getId().getId()); } if (userCredentials.getUserId() != null) { this.userId = toString(userCredentials.getUserId().getId()); @@ -69,8 +69,8 @@ public final class UserCredentialsEntity extends BaseSqlEntity @Override public UserCredentials toData() { - UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(this.getUuid())); - userCredentials.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(getId())); + userCredentials.setCreatedTime(UUIDs.unixTimestamp(getId())); if (userId != null) { userCredentials.setUserId(new UserId(toUUID(userId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java index 3af671583e..0100110daa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java @@ -81,7 +81,7 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< public UserEntity(User user) { if (user.getId() != null) { - this.setUuid(user.getId().getId()); + this.setId(user.getId().getId()); } this.authority = user.getAuthority(); if (user.getTenantId() != null) { @@ -108,8 +108,8 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< @Override public User toData() { - User user = new User(new UserId(this.getUuid())); - user.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + User user = new User(new UserId(getId())); + user.setCreatedTime(UUIDs.unixTimestamp(getId())); user.setAuthority(authority); if (tenantId != null) { user.setTenantId(new TenantId(fromString(tenantId))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java index 0cb69b9d8b..8e81b30e8f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java @@ -62,7 +62,7 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement public WidgetTypeEntity(WidgetType widgetType) { if (widgetType.getId() != null) { - this.setUuid(widgetType.getId().getId()); + this.setId(widgetType.getId().getId()); } if (widgetType.getTenantId() != null) { this.tenantId = toString(widgetType.getTenantId().getId()); @@ -75,8 +75,8 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement @Override public WidgetType toData() { - WidgetType widgetType = new WidgetType(new WidgetTypeId(this.getUuid())); - widgetType.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); + WidgetType widgetType = new WidgetType(new WidgetTypeId(getId())); + widgetType.setCreatedTime(UUIDs.unixTimestamp(getId())); if (tenantId != null) { widgetType.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java index 75ea383090..7df6ea9fe7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java @@ -55,7 +55,7 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl public WidgetsBundleEntity(WidgetsBundle widgetsBundle) { if (widgetsBundle.getId() != null) { - this.setUuid(widgetsBundle.getId().getId()); + this.setId(widgetsBundle.getId().getId()); } if (widgetsBundle.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(widgetsBundle.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index edc9c97a9a..d3ce07f901 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -58,8 +58,8 @@ public abstract class JpaAbstractDao, D> } setSearchText(entity); log.debug("Saving entity {}", entity); - if (entity.getUuid() == null) { - entity.setUuid(UUIDs.timeBased()); + if (entity.getId() == null) { + entity.setId(UUIDs.timeBased()); } entity = getCrudRepository().save(entity); return DaoUtil.getData(entity); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 076d43b0ad..2f3c72d01b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -38,7 +38,7 @@ public interface AlarmRepository extends CrudRepository { @Param("alarmType") String alarmType, Pageable pageable); - @Query(value = "SELECT new org.thingsboard.server.dao.model.sql.AlarmInfoEntity(a) FROM AlarmEntity a, " + + @Query("SELECT new org.thingsboard.server.dao.model.sql.AlarmInfoEntity(a) FROM AlarmEntity a, " + "RelationEntity re " + "WHERE a.tenantId = :tenantId " + "AND a.id = re.toId AND re.toType = 'ALARM' " + @@ -51,21 +51,7 @@ public interface AlarmRepository extends CrudRepository { "AND (:idOffset IS NULL OR a.id < :idOffset) " + "AND (LOWER(a.type) LIKE LOWER(CONCAT(:searchText, '%'))" + "OR LOWER(a.severity) LIKE LOWER(CONCAT(:searchText, '%'))" + - "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))", - countQuery = "SELECT count(a) FROM AlarmEntity a, " + - "RelationEntity re " + - "WHERE a.tenantId = :tenantId " + - "AND a.id = re.toId AND re.toType = 'ALARM' " + - "AND re.relationTypeGroup = 'ALARM' " + - "AND re.relationType = :relationType " + - "AND re.fromId = :affectedEntityId " + - "AND re.fromType = :affectedEntityType " + - "AND (:startId IS NULL OR a.id >= :startId) " + - "AND (:endId IS NULL OR a.id <= :endId) " + - "AND (:idOffset IS NULL OR a.id < :idOffset) " + - "AND (LOWER(a.type) LIKE LOWER(CONCAT(:searchText, '%'))" + - "OR LOWER(a.severity) LIKE LOWER(CONCAT(:searchText, '%'))" + - "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))") + "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))") Page findAlarms(@Param("tenantId") String tenantId, @Param("affectedEntityId") String affectedEntityId, @Param("affectedEntityType") String affectedEntityType, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 56ae6a5000..002c8969da 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.asset; import com.google.common.util.concurrent.ListenableFuture; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; @@ -42,6 +43,7 @@ import java.util.UUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUIDs; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Valerii Sosliuk on 5/19/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java index 371a627818..38abc36940 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java @@ -32,8 +32,6 @@ import org.thingsboard.server.dao.model.sql.AuditLogEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; -import javax.persistence.criteria.Predicate; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.UUID; @@ -41,7 +39,6 @@ import java.util.UUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; import static org.thingsboard.server.dao.DaoUtil.endTimeToId; import static org.thingsboard.server.dao.DaoUtil.startTimeToId; -import static org.thingsboard.server.dao.model.ModelConstants.ID_PROPERTY; @Component @SqlDao diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java index d8272e3a57..c634e5a064 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java @@ -48,17 +48,17 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com } catch (Throwable throwable) { transactionManager.rollback(insertTransaction); if (throwable.getCause() instanceof ConstraintViolationException) { - log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getUuid(), entity.getName(), entity.getType()); + log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getId(), entity.getName(), entity.getType()); TransactionStatus transaction = getTransactionStatus(TransactionDefinition.PROPAGATION_REQUIRES_NEW); try { componentDescriptorEntity = processSaveOrUpdate(entity, insertOrUpdateOnUniqueKeyConflict); } catch (Throwable th) { - log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); + log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); transactionManager.rollback(transaction); } transactionManager.commit(transaction); } else { - log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); + log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); } } return componentDescriptorEntity; @@ -69,7 +69,7 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com protected Query getQuery(ComponentDescriptorEntity entity, String query) { return entityManager.createNativeQuery(query, ComponentDescriptorEntity.class) - .setParameter("id", UUIDConverter.fromTimeUUID(entity.getUuid())) + .setParameter("id", UUIDConverter.fromTimeUUID(entity.getId())) .setParameter("actions", entity.getActions()) .setParameter("clazz", entity.getClazz()) .setParameter("configuration_descriptor", entity.getConfigurationDescriptor().toString()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java index c33dd4f5e1..e281bdeec4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java @@ -40,7 +40,7 @@ public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDe @Override protected ComponentDescriptorEntity doProcessSaveOrUpdate(ComponentDescriptorEntity entity, String query) { getQuery(entity, query).executeUpdate(); - return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getUuid())); + return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getId())); } private static String getInsertString(String conflictStatement) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java index ab4f408da5..67f25c2325 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.component; import com.datastax.driver.core.utils.UUIDs; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -34,9 +35,11 @@ import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Objects; import java.util.Optional; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Valerii Sosliuk on 5/6/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 0361c8d8dd..e859773bf7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.customer; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; @@ -28,10 +29,12 @@ import org.thingsboard.server.dao.model.sql.CustomerEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.UUID; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Valerii Sosliuk on 5/6/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java index e693e81ae1..7b0e6a8882 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java @@ -41,6 +41,9 @@ import java.util.UUID; @SqlDao public class JpaDashboardInfoDao extends JpaAbstractSearchTextDao implements DashboardInfoDao { + @Autowired + private RelationDao relationDao; + @Autowired private DashboardInfoRepository dashboardInfoRepository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index c409f0a91f..0d8d87f74e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -17,13 +17,10 @@ package org.thingsboard.server.dao.sql.entityview; import com.google.common.util.concurrent.ListenableFuture; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.EntitySubtype; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.EntityViewInfo; -import org.thingsboard.server.common.data.UUIDConverter; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -42,6 +39,7 @@ import java.util.Optional; import java.util.UUID; import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; /** * Created by Victor Basanets on 8/31/2017. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java index 8a341e0f81..59f0da89e0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java @@ -69,14 +69,13 @@ public abstract class AbstractEventInsertRepository implements EventInsertReposi protected Query getQuery(EventEntity entity, String query) { return entityManager.createNativeQuery(query, EventEntity.class) - .setParameter("id", UUIDConverter.fromTimeUUID(entity.getUuid())) + .setParameter("id", UUIDConverter.fromTimeUUID(entity.getId())) .setParameter("body", entity.getBody().toString()) .setParameter("entity_id", entity.getEntityId()) .setParameter("entity_type", entity.getEntityType().name()) .setParameter("event_type", entity.getEventType()) .setParameter("event_uid", entity.getEventUid()) - .setParameter("tenant_id", entity.getTenantId()) - .setParameter("ts", entity.getTs()); + .setParameter("tenant_id", entity.getTenantId()); } private EventEntity processSaveOrUpdate(EventEntity entity, String query) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java index f5a00fe7f7..440c618667 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java @@ -40,11 +40,11 @@ public class HsqlEventInsertRepository extends AbstractEventInsertRepository { @Override protected EventEntity doProcessSaveOrUpdate(EventEntity entity, String query) { getQuery(entity, query).executeUpdate(); - return entityManager.find(EventEntity.class, UUIDConverter.fromTimeUUID(entity.getUuid())); + return entityManager.find(EventEntity.class, UUIDConverter.fromTimeUUID(entity.getId())); } private static String getInsertString(String conflictStatement) { - return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id, event.ts = I.ts" + - " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id, I.ts)"; + return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id" + + " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id)"; } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index 528229bb33..c5a6ea2717 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -141,7 +141,7 @@ public class JpaBaseEventDao extends JpaAbstractSearchTimeDao - DaoUtil.toPageData(relationRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable))); + DaoUtil.toPageData(relationRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable))); } private Specification getEntityFieldsSpec(EntityId from, String relationType, RelationTypeGroup typeGroup, EntityType childType) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 3a4cbd84e0..d2dbe725cd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.rule; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.UUIDConverter; @@ -29,9 +30,11 @@ import org.thingsboard.server.dao.rule.RuleChainDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Objects; import java.util.UUID; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; @Slf4j @Component diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java index 7fc55c5752..826d60b119 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java @@ -16,9 +16,11 @@ package org.thingsboard.server.dao.sql.tenant; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -28,8 +30,11 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Objects; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID_STR; + /** * Created by Valerii Sosliuk on 4/30/2017. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 296730e55f..9d39a09404 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -30,6 +30,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.user.UserDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Objects; import java.util.UUID; @@ -83,5 +84,6 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple Objects.toString(pageLink.getTextSearch(), ""), Authority.CUSTOMER_USER, DaoUtil.toPageable(pageLink))); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 6f7490277a..c24e3d8e27 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.widget; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.UUIDConverter; @@ -29,6 +30,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.widget.WidgetsBundleDao; +import java.util.List; import java.util.Objects; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index fafe05957f..c4ac4e9fd8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -150,8 +150,8 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq keyId, query.getStartTs(), query.getEndTs(), - PageRequest.of(0, query.getLimit(), - Sort.by(Sort.Direction.fromString( + new PageRequest(0, query.getLimit(), + new Sort(Sort.Direction.fromString( query.getOrderBy()), "ts"))); tsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(query.getKey())); return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 0a2e210648..bf4cf5d9e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -109,8 +109,8 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements keyId, query.getStartTs(), query.getEndTs(), - PageRequest.of(0, query.getLimit(), - Sort.by(Sort.Direction.fromString( + new PageRequest(0, query.getLimit(), + new Sort(Sort.Direction.fromString( query.getOrderBy()), "ts"))); timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey)); return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index b63e4f7846..efac1cfc42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -39,6 +39,8 @@ import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; +import java.util.List; + import static org.thingsboard.server.dao.service.Validator.validateId; @Service @@ -123,7 +125,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override public void deleteTenants() { log.trace("Executing deleteTenants"); - tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID), DEFAULT_TENANT_REGION); + tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID),DEFAULT_TENANT_REGION); } private DataValidator tenantValidator = @@ -137,31 +139,19 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe validateEmail(tenant.getEmail()); } } - - @Override - protected void validateUpdate(TenantId tenantId, Tenant tenant) { - Tenant old = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); - if (old == null) { - throw new DataValidationException("Can't update non existing tenant!"); - } else if (old.isIsolatedTbRuleEngine() != tenant.isIsolatedTbRuleEngine()) { - throw new DataValidationException("Can't update isolatedTbRuleEngine property!"); - } else if (old.isIsolatedTbCore() != tenant.isIsolatedTbCore()) { - throw new DataValidationException("Can't update isolatedTbCore property!"); - } - } - }; + }; private PaginatedRemover tenantsRemover = new PaginatedRemover() { - @Override - protected PageData findEntities(TenantId tenantId, String region, PageLink pageLink) { - return tenantDao.findTenantsByRegion(tenantId, region, pageLink); - } + @Override + protected PageData findEntities(TenantId tenantId, String region, PageLink pageLink) { + return tenantDao.findTenantsByRegion(tenantId, region, pageLink); + } - @Override - protected void removeEntity(TenantId tenantId, Tenant entity) { - deleteTenant(new TenantId(entity.getUuidId())); - } - }; + @Override + protected void removeEntity(TenantId tenantId, Tenant entity) { + deleteTenant(new TenantId(entity.getUuidId())); + } + }; } diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index 697e34930a..758aaafb10 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -144,7 +144,6 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), - ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -184,9 +183,7 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255), - isolated_tb_core boolean, - isolated_tb_rule_engine boolean + zip varchar(255) ); CREATE TABLE IF NOT EXISTS user_credentials ( @@ -252,4 +249,3 @@ CREATE TABLE IF NOT EXISTS entity_view ( search_text varchar(255), additional_info varchar ); - diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 931856e1df..55893fc124 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -144,7 +144,6 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), - ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -184,9 +183,7 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255), - isolated_tb_core boolean, - isolated_tb_rule_engine boolean + zip varchar(255) ); CREATE TABLE IF NOT EXISTS user_credentials ( @@ -252,28 +249,3 @@ CREATE TABLE IF NOT EXISTS entity_view ( search_text varchar(255), additional_info varchar ); - -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - ttl_ts bigint; - debug_ttl_ts bigint; - ttl_deleted_count bigint DEFAULT 0; - debug_ttl_deleted_count bigint DEFAULT 0; -BEGIN - IF ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; - END IF; - IF debug_ttl > 0 THEN - debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; - END IF; - RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; - RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; - deleted := ttl_deleted_count + debug_ttl_deleted_count; -END -$$; diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index bb0a964c13..32e2a78620 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS $$ diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index f83e80931b..3789444106 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java index e0244c256e..f02ae6b577 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java @@ -19,7 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/dao/src/test/resources/cassandra-test.properties b/dao/src/test/resources/cassandra-test.properties index 51f34a08d6..af9a4b356f 100644 --- a/dao/src/test/resources/cassandra-test.properties +++ b/dao/src/test/resources/cassandra-test.properties @@ -60,4 +60,3 @@ cassandra.query.tenant_rate_limits.enabled=false cassandra.query.tenant_rate_limits.configuration=5000:1,100000:60 cassandra.query.tenant_rate_limits.print_tenant_names=false -service.type=monolith \ No newline at end of file diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 781e0ee01e..c7dd8e6429 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -16,8 +16,6 @@ spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver spring.datasource.hikari.maximumPoolSize = 50 -service.type=monolith - #database.ts.type=timescale #database.ts.type=sql #database.entities.type=sql @@ -35,15 +33,4 @@ service.type=monolith #spring.datasource.password=postgres #spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest #spring.datasource.driverClassName=org.postgresql.Driver -#spring.datasource.hikari.maximumPoolSize = 50 - -queue.core.pack-processing-timeout=3000 -queue.rule-engine.pack-processing-timeout=3000 - -queue.rule-engine.queues[0].name=Main -queue.rule-engine.queues[0].topic=tb_rule_engine.main -queue.rule-engine.queues[0].poll-interval=25 -queue.rule-engine.queues[0].partitions=3 -queue.rule-engine.queues[0].pack-processing-timeout=3000 -queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES -queue.rule-engine.queues[0].submit-strategy.type=BURST \ No newline at end of file +#spring.datasource.hikari.maximumPoolSize = 50 \ No newline at end of file diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index f0facf8cc1..c17b1ddde4 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -19,7 +19,6 @@ const COMPILATION_ERROR = 0; const RUNTIME_ERROR = 1; const TIMEOUT_ERROR = 2; const UNRECOGNIZED = -1; -let headers; const config = require('config'), logger = require('../config/logger')._logger('JsInvokeMessageProcessor'), @@ -44,7 +43,6 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { var responseTopic; try { var request = JSON.parse(message.value.toString('utf8')); - headers = message.headers; var buf = message.headers['requestId']; requestId = Utils.UUIDFromBuffer(buf); buf = message.headers['responseTopic']; @@ -150,8 +148,7 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT messages: [ { key: scriptId, - value: rawResponse, - headers: headers + value: rawResponse } ] } diff --git a/pom.xml b/pom.xml index a880025e3f..7cadb38031 100755 --- a/pom.xml +++ b/pom.xml @@ -30,11 +30,11 @@ ${basedir} thingsboard - 2.2.4.RELEASE - 5.2.2.RELEASE - 5.2.2.RELEASE - 2.2.4.RELEASE - 3.1.0 + 2.1.3.RELEASE + 5.1.5.RELEASE + 5.1.4.RELEASE + 2.1.5.RELEASE + 2.9.0 0.7.0 2.2.0 4.12 @@ -51,12 +51,12 @@ 1.6 2.5 1.4 - 2.10.2 - 2.10.2 - 2.10.2 + 2.9.9.3 + 2.9.9 + 2.9.9 2.2.6 - 2.13 - 2.6.3 + 2.11 + 2.4.2 1.0.2 2.6.2 1.7 @@ -68,7 +68,7 @@ 1.22.1 1.16.18 1.1.0 - 4.1.45.Final + 4.1.37.Final 1.5.0 4.8.0 2.19.1 @@ -77,7 +77,7 @@ 1.0.0 0.7 1.15.0 - 1.64 + 1.56 2.0.1 2.5.0 2.5.3 @@ -92,15 +92,9 @@ 4.1.1 2.57 2.7.7 - 1.25 - 1.3.10 - 1.11.747 - 1.84.0 - 3.2.0 + 1.23 1.5.0 1.4.3 - 1.9.4 - 3.2.2 @@ -897,21 +891,6 @@ jts-core ${jts.version} - - com.amazonaws - aws-java-sdk-sqs - ${amazonaws.sqs.version} - - - com.google.cloud - google-cloud-pubsub - ${pubsub.client.version} - - - com.microsoft.azure - azure-servicebus - ${azure-servicebus.version} - org.passay passay @@ -922,37 +901,6 @@ uap-java ${ua-parser.version} - - commons-beanutils - commons-beanutils - ${commons-beanutils.version} - - - commons-collections - commons-collections - ${commons-collections.version} - - - org.yaml - snakeyaml - ${snakeyaml.version} - - - org.apache.struts - struts-core - ${struts.version} - - - org.apache.struts - struts-taglib - ${struts.version} - - - org.apache.struts - struts-tiles - ${struts.version} - - diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 14c3d4b996..ad58bce619 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -45,7 +45,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -97,6 +97,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -1644,7 +1645,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { addPageLinkToParam(params, pageLink); Map> timeseries = restTemplate.exchange( - baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParamsTs(pageLink), + baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParams(pageLink), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>>() { @@ -1996,24 +1997,12 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } private String getTimeUrlParams(TimePageLink pageLink) { - return getUrlParams(pageLink); - } - - private String getUrlParams(TimePageLink pageLink) { - return getUrlParams(pageLink, "startTime", "endTime"); - } - - private String getUrlParamsTs(TimePageLink pageLink) { - return getUrlParams(pageLink, "startTs", "endTs"); - } - - private String getUrlParams(TimePageLink pageLink, String startTime, String endTime) { - String urlParams = "limit={limit}&ascOrder={ascOrder}"; + String urlParams = this.getUrlParams(pageLink); if (pageLink.getStartTime() != null) { - urlParams += "&" + startTime + "={startTime}"; + urlParams += "&startTime={startTime}"; } if (pageLink.getEndTime() != null) { - urlParams += "&" + endTime + "={endTime}"; + urlParams += "&endTime={endTime}"; } return urlParams; } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java similarity index 54% rename from common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java rename to rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java index 4c25f27a3e..bb0210b1ad 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java @@ -13,25 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.queue; +package org.thingsboard.rule.engine.api; -public interface TbCallback { +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.cluster.ServerAddress; - TbCallback EMPTY = new TbCallback() { +import java.util.function.Consumer; - @Override - public void onSuccess() { +public interface RuleChainTransactionService { - } + void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure); - @Override - public void onFailure(Throwable t) { + void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure); - } - }; - - void onSuccess(); - - void onFailure(Throwable t); + void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] bytes); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java index 7cab689d13..34bfe4d3f4 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java @@ -18,7 +18,6 @@ package org.thingsboard.rule.engine.api; import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -29,11 +28,11 @@ import java.util.UUID; @Builder public final class RuleEngineDeviceRpcRequest { - private final TenantId tenantId; private final DeviceId deviceId; private final int requestId; private final UUID requestUUID; - private final String originServiceId; + private final String originHost; + private final int originPort; private final boolean oneway; private final String method; private final String body; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java index baac594a17..69ce133c5d 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java @@ -16,8 +16,6 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.data.id.DeviceId; - -import java.util.UUID; import java.util.function.Consumer; /** @@ -25,8 +23,8 @@ import java.util.function.Consumer; */ public interface RuleEngineRpcService { - void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body); + void sendRpcReply(DeviceId deviceId, int requestId, String body); - void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest request, Consumer consumer); + void sendRpcRequest(RuleEngineDeviceRpcRequest request, Consumer consumer); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java index d2e19652a2..f57849fa35 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java @@ -44,4 +44,6 @@ public interface RuleEngineTelemetryService { void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, boolean value, FutureCallback callback); + void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes); + } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 9b1edc5f5c..38914f4553 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.alarm.AlarmService; @@ -44,91 +45,27 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import java.util.Set; -import java.util.function.Consumer; /** * Created by ashvayka on 13.01.18. */ public interface TbContext { - /* - * - * METHODS TO CONTROL THE MESSAGE FLOW - * - */ - - /** - * Indicates that message was successfully processed by the rule node. - * Sends message to all Rule Nodes in the Rule Chain - * that are connected to the current Rule Node using "Success" relationType. - * - * @param msg - */ - void tellSuccess(TbMsg msg); - - /** - * Sends message to all Rule Nodes in the Rule Chain - * that are connected to the current Rule Node using specified relationType. - * - * @param msg - * @param relationType - */ void tellNext(TbMsg msg, String relationType); - /** - * Sends message to all Rule Nodes in the Rule Chain - * that are connected to the current Rule Node using one of specified relationTypes. - * - * @param msg - * @param relationTypes - */ + void tellNext(TbMsg msg, String relationType, Throwable th); + void tellNext(TbMsg msg, Set relationTypes); - /** - * Sends message to the current Rule Node with specified delay in milliseconds. - * Note: this message is not queued and may be lost in case of a server restart. - * - * @param msg - */ void tellSelf(TbMsg msg, long delayMs); - /** - * Notifies Rule Engine about failure to process current message. - * - * @param msg - message - * @param th - exception - */ - void tellFailure(TbMsg msg, Throwable th); - - /** - * Puts new message to queue for processing by the Root Rule Chain - * - * @param msg - message - */ - void enqueue(TbMsg msg, Runnable onSuccess, Consumer onFailure); - - /** - * Puts new message to custom queue for processing - * - * @param msg - message - */ - void enqueue(TbMsg msg, String queueName, Runnable onSuccess, Consumer onFailure); - - void enqueueForTellFailure(TbMsg msg, String failureMessage); - - void enqueueForTellNext(TbMsg msg, String relationType); - - void enqueueForTellNext(TbMsg msg, Set relationTypes); - - void enqueueForTellNext(TbMsg msg, String relationType, Runnable onSuccess, Consumer onFailure); - - void enqueueForTellNext(TbMsg msg, Set relationTypes, Runnable onSuccess, Consumer onFailure); + boolean isLocalEntity(EntityId entityId); - void enqueueForTellNext(TbMsg msg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure); + void tellFailure(TbMsg msg, Throwable th); - void enqueueForTellNext(TbMsg msg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure); + void updateSelf(RuleNode self); - void ack(TbMsg tbMsg); + void sendTbMsgToRuleEngine(TbMsg msg); TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data); @@ -140,17 +77,8 @@ public interface TbContext { TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId); - // TODO: Does this changes the message? TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId); - /* - * - * METHODS TO PROCESS THE MESSAGES - * - */ - - boolean isLocalEntity(EntityId entityId); - RuleNodeId getSelfId(); TenantId getTenantId(); @@ -201,7 +129,9 @@ public interface TbContext { void logJsEvalFailure(); - String getServiceId(); + String getNodeId(); + + RuleChainTransactionService getRuleChainTransactionService(); EventLoopGroup getSharedEventLoop(); @@ -209,7 +139,7 @@ public interface TbContext { ResultSetFuture submitCassandraTask(CassandraStatementTask task); - @Deprecated RedisTemplate getRedisTemplate(); + String getServerAddress(); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java index 85fc75139b..26f7d14f8b 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java @@ -16,7 +16,7 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; import java.util.concurrent.ExecutionException; @@ -31,6 +31,6 @@ public interface TbNode { void destroy(); - default void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) {} + default void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) {} } diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 299fdae084..1ada70091c 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -35,7 +35,8 @@ UTF-8 ${basedir}/../.. - 1.11.747 + 1.11.323 + 1.83.0 1.16.0 @@ -98,6 +99,7 @@ com.google.cloud google-cloud-pubsub + ${pubsub.client.version} com.google.api.grpc diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java index b3344105bb..6320d5625f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import javax.script.ScriptException; + import static org.thingsboard.common.util.DonAsynchron.withCallback; @@ -61,18 +63,16 @@ public abstract class TbAbstractAlarmNode ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Created"), - throwable -> ctx.tellFailure(toAlarmMsg(ctx, alarmResult, msg), throwable)); + ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Created"); + ctx.sendTbMsgToRuleEngine(ctx.alarmCreatedMsg(alarmResult.alarm, ctx.getSelfId())); } else if (alarmResult.isUpdated) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Updated"); } else if (alarmResult.isCleared) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Cleared"); - } else { - ctx.tellSuccess(msg); } }, - t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); + t -> ctx.tellFailure(msg, t) + , ctx.getDbCallbackExecutor()); } protected abstract ListenableFuture processAlarm(TbContext ctx, TbMsg msg); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java index bdf8c5fe1e..3ee2127fde 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java @@ -63,7 +63,7 @@ public abstract class TbAbstractCustomerActionNode ctx.tellSuccess(msg), + m -> ctx.tellNext(msg, "Success"), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } @@ -122,9 +122,7 @@ public abstract class TbAbstractCustomerActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), - throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); + ctx.sendTbMsgToRuleEngine(ctx.customerCreatedMsg(savedCustomer, ctx.getSelfId())); return Optional.of(savedCustomer.getId()); } return Optional.empty(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java index 3b18c540e8..5c92c7536c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java @@ -187,9 +187,7 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Device Created message: {}", savedDevice), - throwable -> log.warn("Failed to push Device Created message: {}", savedDevice, throwable)); + ctx.sendTbMsgToRuleEngine(ctx.deviceCreatedMsg(savedDevice, ctx.getSelfId())); targetEntity.setEntityId(savedDevice.getId()); } break; @@ -204,9 +202,7 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Asset Created message: {}", savedAsset), - throwable -> log.warn("Failed to push Asset Created message: {}", savedAsset, throwable)); + ctx.sendTbMsgToRuleEngine(ctx.assetCreatedMsg(savedAsset, ctx.getSelfId())); targetEntity.setEntityId(savedAsset.getId()); } break; @@ -220,9 +216,7 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), - throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); + ctx.sendTbMsgToRuleEngine(ctx.customerCreatedMsg(savedCustomer, ctx.getSelfId())); targetEntity.setEntityId(savedCustomer.getId()); } break; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java index c5dc5cc5f4..f009b737e9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java @@ -89,7 +89,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { if ((endTime != 0 && endTime > now && startTime < now) || (endTime == 0 && startTime < now)) { if (DataConstants.ATTRIBUTES_UPDATED.equals(msg.getType()) || DataConstants.ACTIVITY_EVENT.equals(msg.getType()) || - SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType())) { + SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType()) ) { Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(msg.getData())); List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr.getKey(), entityView)).collect(Collectors.toList()); @@ -117,14 +117,13 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr, entityView)).collect(Collectors.toList()); - if (!filteredAttributes.isEmpty()) { + if (filteredAttributes != null && !filteredAttributes.isEmpty()) { ctx.getAttributesService().removeAll(ctx.getTenantId(), entityView.getId(), scope, filteredAttributes); transformAndTellNext(ctx, msg, entityView); } } } } - ctx.ack(msg); }, t -> ctx.tellFailure(msg, t)); } else { @@ -136,7 +135,8 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } private void transformAndTellNext(TbContext ctx, TbMsg msg, EntityView entityView) { - ctx.enqueueForTellNext(ctx.newMsg(msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()), SUCCESS); + TbMsg updMsg = ctx.transformMsg(msg, msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()); + ctx.tellNext(updMsg, SUCCESS); } private boolean attributeContainsInEntityView(String scope, String attrKey, EntityView entityView) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java index 2831e890d6..62410e3052 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java @@ -58,7 +58,7 @@ public class TbLogNode implements TbNode { toString -> { ctx.logJsEvalResponse(); log.info(toString); - ctx.tellSuccess(msg); + ctx.tellNext(msg, SUCCESS); }, t -> { ctx.logJsEvalResponse(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 19d8515455..2e99e75b53 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -68,19 +68,18 @@ public class TbMsgCountNode implements TbNode { public void onMsg(TbContext ctx, TbMsg msg) { if (msg.getType().equals(TB_MSG_COUNT_NODE_MSG) && msg.getId().equals(nextTickId)) { JsonObject telemetryJson = new JsonObject(); - telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getServiceId(), messagesProcessed.longValue()); + telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getNodeId(), messagesProcessed.longValue()); messagesProcessed = new AtomicLong(0); TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("delta", Long.toString(System.currentTimeMillis() - lastScheduledTs + delay)); - TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, gson.toJson(telemetryJson)); - ctx.enqueueForTellNext(tbMsg, SUCCESS); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, TbMsgDataType.JSON, gson.toJson(telemetryJson), null, null, 0L); + ctx.tellNext(tbMsg, SUCCESS); scheduleTickMsg(ctx); } else { messagesProcessed.incrementAndGet(); - ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 926c3c25f3..a583fe3c74 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -89,8 +89,10 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { - withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + withCallback(save(msg, ctx), aVoid -> { + ctx.tellNext(msg, SUCCESS); + }, e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java index d43ebb26ef..59d27018d7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java @@ -74,8 +74,11 @@ public class TbSnsNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { withCallback(publishMessageAsync(ctx, msg), - ctx::tellSuccess, - t -> ctx.tellFailure(processException(ctx, msg, t), t)); + m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), + t -> { + TbMsg next = processException(ctx, msg, t); + ctx.tellFailure(next, t); + }); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java index 3cc6074165..37f4e4baee 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java @@ -80,10 +80,13 @@ public class TbSqsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { withCallback(publishMessageAsync(ctx, msg), - ctx::tellSuccess, - t -> ctx.tellFailure(processException(ctx, msg, t), t)); + m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), + t -> { + TbMsg next = processException(ctx, msg, t); + ctx.tellFailure(next, t); + }); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index 46ef675ce1..e16b2f2f0e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -74,7 +74,7 @@ public class TbMsgGeneratorNode implements TbNode { } @Override - public void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) { + public void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) { updateGeneratorState(ctx); } @@ -97,7 +97,7 @@ public class TbMsgGeneratorNode implements TbNode { withCallback(generate(ctx), m -> { if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { - ctx.enqueueForTellNext(m, SUCCESS); + ctx.tellNext(m, SUCCESS); scheduleTickMsg(ctx); currentMsgCount++; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java index 1807c5c4fe..0ef9c4d46a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java @@ -65,16 +65,15 @@ public class TbMsgDelayNode implements TbNode { if (msg.getType().equals(TB_MSG_DELAY_NODE_MSG)) { TbMsg pendingMsg = pendingMsgs.remove(UUID.fromString(msg.getData())); if (pendingMsg != null) { - ctx.enqueueForTellNext(pendingMsg, SUCCESS); + ctx.tellNext(pendingMsg, SUCCESS); } } else { - if (pendingMsgs.size() < config.getMaxPendingMsgs()) { + if(pendingMsgs.size() < config.getMaxPendingMsgs()) { pendingMsgs.put(msg.getId(), msg); TbMsg tickMsg = ctx.newMsg(TB_MSG_DELAY_NODE_MSG, ctx.getSelfId(), new TbMsgMetaData(), msg.getId().toString()); ctx.tellSelf(tickMsg, getDelay(msg)); - ctx.ack(msg); } else { - ctx.tellFailure(msg, new RuntimeException("Max limit of pending messages reached!")); + ctx.tellNext(msg, FAILURE, new RuntimeException("Max limit of pending messages reached!")); } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java index dc0eb9f396..086e3aded0 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java @@ -72,13 +72,14 @@ public class TbCheckAlarmStatusNode implements TbNode { break; } } + if (isPresent) { ctx.tellNext(msg, "True"); } else { ctx.tellNext(msg, "False"); } } else { - ctx.tellFailure(msg, new TbNodeException("No such alarm found.")); + ctx.tellFailure(msg, new TbNodeException("No such Alarm found.")); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java index e5e84ba0b6..59c66dfb8b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java @@ -44,7 +44,7 @@ public class TbMsgTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { ctx.tellNext(msg, config.getMessageTypes().contains(msg.getType()) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java index 07a0508a87..d0fe593711 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java @@ -45,7 +45,7 @@ public class TbMsgTypeSwitchNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { String relationType; if (msg.getType().equals(SessionMsgType.POST_ATTRIBUTES_REQUEST.name())) { relationType = "Post attributes"; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java index 5dd11c9318..486f9fbd84 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java @@ -42,7 +42,7 @@ public class TbOriginatorTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { EntityType originatorType = msg.getOriginator().getEntityType(); ctx.tellNext(msg, config.getOriginatorTypes().contains(originatorType) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java deleted file mode 100644 index b5af3f1563..0000000000 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.flow; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.rule.engine.api.EmptyNodeConfiguration; -import org.thingsboard.rule.engine.api.RuleNode; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNode; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.server.common.data.plugin.ComponentType; -import org.thingsboard.server.common.msg.TbMsg; - -@Slf4j -@RuleNode( - type = ComponentType.ACTION, - name = "acknowledge", - configClazz = EmptyNodeConfiguration.class, - nodeDescription = "Acknowledges the incoming message", - nodeDetails = "After acknowledgement, the message is pushed to related rule nodes. Useful if you don't care what happens to this message next.", - uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbNodeEmptyConfig" -) -public class TbAckNode implements TbNode { - - EmptyNodeConfiguration config; - - @Override - public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); - } - - @Override - public void onMsg(TbContext ctx, TbMsg msg) { - ctx.ack(msg); - ctx.tellSuccess(msg); - } - - @Override - public void destroy() { - } -} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java deleted file mode 100644 index d5048eadbd..0000000000 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.flow; - -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.rule.engine.api.RuleNode; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNode; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.rule.engine.api.TbRelationTypes; -import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.server.common.data.plugin.ComponentType; -import org.thingsboard.server.common.msg.TbMsg; - -import static org.thingsboard.common.util.DonAsynchron.withCallback; - -@Slf4j -@RuleNode( - type = ComponentType.ACTION, - name = "checkpoint", - configClazz = TbCheckpointNodeConfiguration.class, - nodeDescription = "transfers the message to another queue", - nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.", - uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbActionNodeCheckPointConfig" -) -public class TbCheckpointNode implements TbNode { - - private TbCheckpointNodeConfiguration config; - - @Override - public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, TbCheckpointNodeConfiguration.class); - } - - @Override - public void onMsg(TbContext ctx, TbMsg msg) { - ctx.enqueueForTellNext(msg, config.getQueueName(), TbRelationTypes.SUCCESS, () -> ctx.ack(msg), error -> ctx.tellFailure(msg, error)); - } - - @Override - public void destroy() { - } -} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java index 5a8b92cf80..8e3bcdb7dc 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java @@ -99,7 +99,7 @@ public class TbPubSubNode implements TbNode { ApiFutures.addCallback(messageIdFuture, new ApiFutureCallback() { public void onSuccess(String messageId) { TbMsg next = processPublishResult(ctx, msg, messageId); - ctx.tellSuccess(next); + ctx.tellNext(next, TbRelationTypes.SUCCESS); } public void onFailure(Throwable t) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java index 832a86b5cb..fe765abdf9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java @@ -47,12 +47,11 @@ import java.util.concurrent.TimeoutException; type = ComponentType.ACTION, name = "gps geofencing events", configClazz = TbGpsGeofencingActionNodeConfiguration.class, - relationTypes = {"Success", "Entered", "Left", "Inside", "Outside"}, + relationTypes = {"Entered", "Left", "Inside", "Outside"}, nodeDescription = "Produces incoming messages using GPS based geofencing", nodeDetails = "Extracts latitude and longitude parameters from incoming message and returns different events based on configuration parameters", uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbActionNodeGpsGeofencingConfig" -) + configDirective = "tbActionNodeGpsGeofencingConfig") public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { private final Map entityStates = new HashMap<>(); @@ -67,7 +66,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { try { Optional entry = ctx.getAttributesService() - .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getServiceId()) + .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getNodeId()) .get(1, TimeUnit.MINUTES); if (entry.isPresent()) { JsonObject element = parser.parse(entry.get().getValueAsString()).getAsJsonObject(); @@ -79,26 +78,17 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode (entityState.isInside() ? - TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { - setStaid(ctx, msg.getOriginator(), entityState); - ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); - told = true; - } + } else if (!entityState.isStayed()) { + long stayTime = ts - entityState.getStateSwitchTime(); + if (stayTime > (entityState.isInside() ? + TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { + setStaid(ctx, msg.getOriginator(), entityState); + ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); } } - if (!told) { - ctx.tellSuccess(msg); - } } private void switchState(TbContext ctx, EntityId entityId, EntityGeofencingState entityState, boolean matches, long ts) { @@ -118,7 +108,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode attributeKvEntryList = Collections.singletonList(entry); ctx.getAttributesService().save(ctx.getTenantId(), entityId, DataConstants.SERVER_SCOPE, attributeKvEntryList); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java index 4955ebc400..267e8711aa 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java @@ -17,21 +17,12 @@ package org.thingsboard.rule.engine.kafka; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.BooleanUtils; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.*; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.header.internals.RecordHeaders; -import org.thingsboard.rule.engine.api.RuleNode; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNode; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.rule.engine.api.TbRelationTypes; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -39,6 +30,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Properties; +import java.util.concurrent.ExecutionException; @Slf4j @RuleNode( @@ -71,7 +63,7 @@ public class TbKafkaNode implements TbNode { public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { this.config = TbNodeUtils.convert(configuration, TbKafkaNodeConfiguration.class); Properties properties = new Properties(); - properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getServiceId()); + properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getNodeId()); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getBootstrapServers()); properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, config.getValueSerializer()); properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, config.getKeySerializer()); @@ -81,7 +73,8 @@ public class TbKafkaNode implements TbNode { properties.put(ProducerConfig.LINGER_MS_CONFIG, config.getLinger()); properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, config.getBufferMemory()); if (config.getOtherProperties() != null) { - config.getOtherProperties().forEach(properties::put); + config.getOtherProperties() + .forEach(properties::put); } addMetadataKeyValuesAsKafkaHeaders = BooleanUtils.toBooleanDefaultIfNull(config.isAddMetadataKeyValuesAsKafkaHeaders(), false); toBytesCharset = config.getKafkaHeadersCharset() != null ? Charset.forName(config.getKafkaHeadersCharset()) : StandardCharsets.UTF_8; @@ -93,7 +86,7 @@ public class TbKafkaNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { String topic = TbNodeUtils.processPattern(config.getTopicPattern(), msg.getMetaData()); try { if (!addMetadataKeyValuesAsKafkaHeaders) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java index ee46b81116..1c41c2a124 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.util.Properties; import static org.thingsboard.common.util.DonAsynchron.withCallback; +import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @Slf4j @RuleNode( @@ -78,7 +79,7 @@ public class TbSendEmailNode implements TbNode { sendEmail(ctx, email); return null; }), - ok -> ctx.tellSuccess(msg), + ok -> ctx.tellNext(msg, SUCCESS), fail -> ctx.tellFailure(msg, fail)); } catch (Exception ex) { ctx.tellFailure(msg, ex); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 8aa15a4f38..0bf1c23cc7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -43,6 +43,7 @@ import java.util.stream.Collectors; import static org.thingsboard.common.util.DonAsynchron.withCallback; import static org.thingsboard.rule.engine.api.TbRelationTypes.FAILURE; +import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; import static org.thingsboard.server.common.data.DataConstants.CLIENT_SCOPE; import static org.thingsboard.server.common.data.DataConstants.LATEST_TS; import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; @@ -100,7 +101,7 @@ public abstract class TbAbstractGetAttributesNode ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java index 43738fd706..2906d3922f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java @@ -59,7 +59,7 @@ public abstract class TbAbstractGetEntityDetailsNode ctx.tellNext(m, SUCCESS), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java index c7387a5ed0..bc19d6c56f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java @@ -51,7 +51,8 @@ public abstract class TbEntityGetAttrNode implements TbNode @Override public void onMsg(TbContext ctx, TbMsg msg) { try { - withCallback(findEntityAsync(ctx, msg.getOriginator()), + withCallback( + findEntityAsync(ctx, msg.getOriginator()), entityId -> safeGetAttributes(ctx, msg, entityId), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { @@ -88,7 +89,7 @@ public abstract class TbEntityGetAttrNode implements TbNode String attrName = config.getAttrMapping().get(r.getKey()); msg.getMetaData().putValue(attrName, r.getValueAsString()); }); - ctx.tellSuccess(msg); + ctx.tellNext(msg, SUCCESS); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java index 03f0652c3f..653b571b4b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.msg.TbMsg; @Slf4j @RuleNode(type = ComponentType.ENRICHMENT, - name = "related device attributes", + name = "device attributes", configClazz = TbGetDeviceAttrNodeConfiguration.class, nodeDescription = "Add Originators Related Device Attributes and Latest Telemetry value into Message Metadata", nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message metadata " + diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java index 12577076d2..717cb207c4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java @@ -54,10 +54,10 @@ public class TbGetOriginatorFieldsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { try { withCallback(putEntityFields(ctx, msg.getOriginator(), msg), - i -> ctx.tellSuccess(msg), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); + i -> ctx.tellNext(msg, SUCCESS), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { ctx.tellFailure(msg, th); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index a3d84b25db..caef71a4db 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -106,7 +106,8 @@ public class TbGetTelemetryNode implements TbNode { ListenableFuture> list = ctx.getTimeseriesService().findAll(ctx.getTenantId(), msg.getOriginator(), buildQueries(msg)); DonAsynchron.withCallback(list, data -> { process(data, msg); - ctx.tellSuccess(ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData())); + TbMsg newMsg = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData()); + ctx.tellNext(newMsg, SUCCESS); }, error -> ctx.tellFailure(msg, error), ctx.getDbCallbackExecutor()); } catch (Exception e) { ctx.tellFailure(msg, e); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java index 7e27ec496e..6d45a3a05c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java @@ -72,12 +72,12 @@ public class TbMqttNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { String topic = TbNodeUtils.processPattern(this.config.getTopicPattern(), msg.getMetaData()); this.mqttClient.publish(topic, Unpooled.wrappedBuffer(msg.getData().getBytes(UTF8)), MqttQoS.AT_LEAST_ONCE) .addListener(future -> { if (future.isSuccess()) { - ctx.tellSuccess(msg); + ctx.tellNext(msg, TbRelationTypes.SUCCESS); } else { TbMsg next = processException(ctx, msg, future.cause()); ctx.tellFailure(next, future.cause()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java index 4a61cf63f3..3c3bde2915 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java @@ -74,9 +74,9 @@ public class TbRabbitMqNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) { + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { withCallback(publishMessageAsync(ctx, msg), - ctx::tellSuccess, + m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), t -> { TbMsg next = processException(ctx, msg, t); ctx.tellFailure(next, t); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index ba860c103b..65eea0b4c9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.http.client.Netty4ClientHttpRequestFactory; import org.springframework.util.concurrent.ListenableFuture; @@ -83,7 +84,7 @@ class TbHttpClient { } } - void processMessage(TbContext ctx, TbMsg msg) { + void processMessage(TbContext ctx, TbMsg msg, TbRedisQueueProcessor queueProcessor) { String endpointUrl = TbNodeUtils.processPattern(config.getRestEndpointUrlPattern(), msg.getMetaData()); HttpHeaders headers = prepareHeaders(msg.getMetaData()); HttpMethod method = HttpMethod.valueOf(config.getRequestMethod()); @@ -94,6 +95,13 @@ class TbHttpClient { future.addCallback(new ListenableFutureCallback>() { @Override public void onFailure(Throwable throwable) { + if (config.isUseRedisQueueForMsgPersistence()) { + if (throwable instanceof HttpClientErrorException) { + processHttpClientError(((HttpClientErrorException) throwable).getStatusCode(), msg, queueProcessor); + } else { + queueProcessor.pushOnFailure(msg); + } + } TbMsg next = processException(ctx, msg, throwable); ctx.tellFailure(next, throwable); } @@ -101,9 +109,15 @@ class TbHttpClient { @Override public void onSuccess(ResponseEntity responseEntity) { if (responseEntity.getStatusCode().is2xxSuccessful()) { + if (config.isUseRedisQueueForMsgPersistence()) { + queueProcessor.resetCounter(); + } TbMsg next = processResponse(ctx, msg, responseEntity); - ctx.tellSuccess(next); + ctx.tellNext(next, TbRelationTypes.SUCCESS); } else { + if (config.isUseRedisQueueForMsgPersistence()) { + processHttpClientError(responseEntity.getStatusCode(), msg, queueProcessor); + } TbMsg next = processFailureResponse(ctx, msg, responseEntity); ctx.tellNext(next, TbRelationTypes.FAILURE); } @@ -169,4 +183,11 @@ class TbHttpClient { } } + private void processHttpClientError(HttpStatus statusCode, TbMsg msg, TbRedisQueueProcessor queueProcessor) { + if (statusCode.is4xxClientError()) { + log.warn("[{}] Client error during message delivering!", msg); + } else { + queueProcessor.pushOnFailure(msg); + } + } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java new file mode 100644 index 0000000000..cff9faaab5 --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.rest; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.ListOperations; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.server.common.msg.TbMsg; + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Data +@Slf4j +class TbRedisQueueProcessor { + + private static final int MAX_QUEUE_SIZE = Integer.MAX_VALUE; + + private final TbContext ctx; + private final TbHttpClient httpClient; + private final ExecutorService executor; + private final ListOperations listOperations; + private final String redisKey; + private final boolean trimQueue; + private final int maxQueueSize; + + private AtomicInteger failuresCounter; + private Future future; + + TbRedisQueueProcessor(TbContext ctx, TbHttpClient httpClient, boolean trimQueue, int maxQueueSize) { + this.ctx = ctx; + this.httpClient = httpClient; + this.executor = Executors.newSingleThreadExecutor(); + this.listOperations = ctx.getRedisTemplate().opsForList(); + this.redisKey = constructRedisKey(); + this.trimQueue = trimQueue; + this.maxQueueSize = maxQueueSize; + init(); + } + + private void init() { + failuresCounter = new AtomicInteger(0); + future = executor.submit(() -> { + while (true) { + if (failuresCounter.get() != 0 && failuresCounter.get() % 50 == 0) { + sleep("Target HTTP server is down...", 3); + } + if (listOperations.size(redisKey) > 0) { + List list = listOperations.range(redisKey, -10, -1); + list.forEach(obj -> { + TbMsg msg = TbMsg.fromBytes((byte[]) obj); + log.debug("Trying to send the message: {}", msg); + listOperations.remove(redisKey, -1, obj); + httpClient.processMessage(ctx, msg, this); + }); + } else { + sleep("Queue is empty, waiting for tasks!", 1); + } + } + }); + } + + void destroy() { + if (future != null) { + future.cancel(true); + } + if (executor != null) { + executor.shutdownNow(); + } + } + + void push(TbMsg msg) { + listOperations.leftPush(redisKey, TbMsg.toByteArray(msg)); + if (trimQueue) { + listOperations.trim(redisKey, 0, validateMaxQueueSize()); + } + } + + void pushOnFailure(TbMsg msg) { + listOperations.rightPush(redisKey, TbMsg.toByteArray(msg)); + failuresCounter.incrementAndGet(); + } + + void resetCounter() { + failuresCounter.set(0); + } + + private String constructRedisKey() { + return ctx.getServerAddress() + ctx.getSelfId(); + } + + private int validateMaxQueueSize() { + if (maxQueueSize != 0) { + return maxQueueSize; + } + return MAX_QUEUE_SIZE; + } + + private void sleep(String logMessage, int sleepSeconds) { + try { + log.debug(logMessage); + TimeUnit.SECONDS.sleep(sleepSeconds); + } catch (InterruptedException e) { + throw new IllegalStateException("Thread interrupted!", e); + } + } +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java index 9cb171d0dc..9b9d275977 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java @@ -25,6 +25,8 @@ import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; +import java.util.concurrent.ExecutionException; + @Slf4j @RuleNode( type = ComponentType.EXTERNAL, @@ -45,6 +47,7 @@ public class TbRestApiCallNode implements TbNode { private boolean useRedisQueueForMsgPersistence; private TbHttpClient httpClient; + private TbRedisQueueProcessor queueProcessor; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -52,13 +55,20 @@ public class TbRestApiCallNode implements TbNode { httpClient = new TbHttpClient(config); useRedisQueueForMsgPersistence = config.isUseRedisQueueForMsgPersistence(); if (useRedisQueueForMsgPersistence) { - log.warn("[{}][{}] Usage of Redis Template is deprecated starting 2.5 and will have no affect", ctx.getTenantId(), ctx.getSelfId()); + if (ctx.getRedisTemplate() == null) { + throw new RuntimeException("Redis cache type must be used!"); + } + queueProcessor = new TbRedisQueueProcessor(ctx, httpClient, config.isTrimQueue(), config.getMaxQueueSize()); } } @Override - public void onMsg(TbContext ctx, TbMsg msg) { - httpClient.processMessage(ctx, msg); + public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + if (useRedisQueueForMsgPersistence) { + queueProcessor.push(msg); + } else { + httpClient.processMessage(ctx, msg, null); + } } @Override @@ -66,6 +76,9 @@ public class TbRestApiCallNode implements TbNode { if (this.httpClient != null) { this.httpClient.destroy(); } + if (this.queueProcessor != null) { + this.queueProcessor.destroy(); + } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 145e73d450..9444b52210 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -28,8 +28,6 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import java.util.UUID; - @Slf4j @RuleNode( type = ComponentType.ACTION, @@ -52,22 +50,15 @@ public class TbSendRPCReplyNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { - String serviceIdStr = msg.getMetaData().getValue(config.getServiceIdMetaDataAttribute()); - String sessionIdStr = msg.getMetaData().getValue(config.getSessionIdMetaDataAttribute()); String requestIdStr = msg.getMetaData().getValue(config.getRequestIdMetaDataAttribute()); if (msg.getOriginator().getEntityType() != EntityType.DEVICE) { ctx.tellFailure(msg, new RuntimeException("Message originator is not a device entity!")); } else if (StringUtils.isEmpty(requestIdStr)) { ctx.tellFailure(msg, new RuntimeException("Request id is not present in the metadata!")); - } else if (StringUtils.isEmpty(serviceIdStr)) { - ctx.tellFailure(msg, new RuntimeException("Service id is not present in the metadata!")); - } else if (StringUtils.isEmpty(sessionIdStr)) { - ctx.tellFailure(msg, new RuntimeException("Session id is not present in the metadata!")); } else if (StringUtils.isEmpty(msg.getData())) { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { - ctx.getRpcService().sendRpcReplyToDevice(serviceIdStr, UUID.fromString(sessionIdStr), Integer.parseInt(requestIdStr), msg.getData()); - ctx.tellSuccess(msg); + ctx.getRpcService().sendRpcReply(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java index ad1d399e76..9aa94905e6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java @@ -16,6 +16,8 @@ package org.thingsboard.rule.engine.rpc; import com.datastax.driver.core.utils.UUIDs; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.JsonNode; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -36,6 +38,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; +import java.io.IOException; import java.util.Random; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -83,8 +86,10 @@ public class TbSendRPCRequestNode implements TbNode { tmp = msg.getMetaData().getValue("requestUUID"); UUID requestUUID = !StringUtils.isEmpty(tmp) ? UUID.fromString(tmp) : UUIDs.timeBased(); - tmp = msg.getMetaData().getValue("originServiceId"); - String originServiceId = !StringUtils.isEmpty(tmp) ? tmp : null; + tmp = msg.getMetaData().getValue("originHost"); + String originHost = !StringUtils.isEmpty(tmp) ? tmp : null; + tmp = msg.getMetaData().getValue("originPort"); + int originPort = !StringUtils.isEmpty(tmp) ? Integer.parseInt(tmp) : 0; tmp = msg.getMetaData().getValue("expirationTime"); long expirationTime = !StringUtils.isEmpty(tmp) ? Long.parseLong(tmp) : (System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(config.getTimeoutInSeconds())); @@ -101,25 +106,24 @@ public class TbSendRPCRequestNode implements TbNode { .oneway(oneway) .method(json.get("method").getAsString()) .body(params) - .tenantId(ctx.getTenantId()) .deviceId(new DeviceId(msg.getOriginator().getId())) .requestId(requestId) .requestUUID(requestUUID) - .originServiceId(originServiceId) + .originHost(originHost) + .originPort(originPort) .expirationTime(expirationTime) .restApiCall(restApiCall) .build(); - ctx.getRpcService().sendRpcRequestToDevice(request, ruleEngineDeviceRpcResponse -> { + ctx.getRpcService().sendRpcRequest(request, ruleEngineDeviceRpcResponse -> { if (!ruleEngineDeviceRpcResponse.getError().isPresent()) { - TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); - ctx.enqueueForTellNext(next, TbRelationTypes.SUCCESS); + TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); + ctx.tellNext(next, TbRelationTypes.SUCCESS); } else { - TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); - ctx.enqueueForTellFailure(next, ruleEngineDeviceRpcResponse.getError().get().name()); + TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); + ctx.tellFailure(next, new RuntimeException(ruleEngineDeviceRpcResponse.getError().get().name())); } }); - ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java index bd3a3cdfb2..d01f28f136 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java @@ -16,40 +16,18 @@ package org.thingsboard.rule.engine.rpc; import lombok.Data; -import org.springframework.util.StringUtils; import org.thingsboard.rule.engine.api.NodeConfiguration; import org.thingsboard.server.common.data.DataConstants; @Data public class TbSendRpcReplyNodeConfiguration implements NodeConfiguration { - public static final String SERVICE_ID = "serviceId"; - public static final String SESSION_ID = "sessionId"; - public static final String REQUEST_ID = "requestId"; - - private String serviceIdMetaDataAttribute; - private String sessionIdMetaDataAttribute; private String requestIdMetaDataAttribute; @Override public TbSendRpcReplyNodeConfiguration defaultConfiguration() { TbSendRpcReplyNodeConfiguration configuration = new TbSendRpcReplyNodeConfiguration(); - configuration.setServiceIdMetaDataAttribute(SERVICE_ID); - configuration.setSessionIdMetaDataAttribute(SESSION_ID); - configuration.setRequestIdMetaDataAttribute(REQUEST_ID); + configuration.setRequestIdMetaDataAttribute("requestId"); return configuration; } - - public String getServiceIdMetaDataAttribute() { - return !StringUtils.isEmpty(serviceIdMetaDataAttribute) ? serviceIdMetaDataAttribute : SERVICE_ID; - } - - public String getSessionIdMetaDataAttribute() { - return !StringUtils.isEmpty(sessionIdMetaDataAttribute) ? sessionIdMetaDataAttribute : SESSION_ID; - } - - public String getRequestIdMetaDataAttribute() { - return !StringUtils.isEmpty(requestIdMetaDataAttribute) ? requestIdMetaDataAttribute : REQUEST_ID; - } } - diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index a0970238d9..3ad89bc05a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -61,9 +61,13 @@ public class TbMsgAttributesNode implements TbNode { ctx.tellFailure(msg, new IllegalArgumentException("Unsupported msg type: " + msg.getType())); return; } + String src = msg.getData(); Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(src)); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), config.getScope(), new ArrayList<>(attributes), new TelemetryNodeCallback(ctx, msg)); + if (msg.getOriginator().getEntityType() == EntityType.DEVICE && DataConstants.SHARED_SCOPE.equals(config.getScope())) { + ctx.getTelemetryService().onSharedAttributesUpdate(ctx.getTenantId(), new DeviceId(msg.getOriginator().getId()), attributes); + } } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index bc135caa5e..d735f7f93f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -74,7 +74,7 @@ public class TbMsgTimeseriesNode implements TbNode { } String src = msg.getData(); Map> tsKvMap = JsonConverter.convertToTelemetry(new JsonParser().parse(src), ts); - if (tsKvMap.isEmpty()) { + if (tsKvMap == null) { ctx.tellFailure(msg, new IllegalArgumentException("Msg body is empty: " + src)); return; } @@ -85,7 +85,7 @@ public class TbMsgTimeseriesNode implements TbNode { } } String ttlValue = msg.getMetaData().getValue("TTL"); - long ttl = !StringUtils.isEmpty(ttlValue) ? Long.parseLong(ttlValue) : config.getDefaultTTL(); + long ttl = !StringUtils.isEmpty(ttlValue) ? Long.valueOf(ttlValue) : config.getDefaultTTL(); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), tsKvEntryList, ttl, new TelemetryNodeCallback(ctx, msg)); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java index bd15c5bd83..d545d41d7b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java @@ -22,6 +22,8 @@ import org.thingsboard.server.common.msg.TbMsg; import javax.annotation.Nullable; +import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; + /** * Created by ashvayka on 02.04.18. */ @@ -32,7 +34,7 @@ class TelemetryNodeCallback implements FutureCallback { @Override public void onSuccess(@Nullable Void result) { - ctx.tellSuccess(msg); + ctx.tellNext(msg, SUCCESS); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index 751b061666..5423236499 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -25,6 +25,10 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgTransactionData; + +import java.util.concurrent.ExecutionException; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @@ -33,23 +37,37 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization start", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", + nodeDescription = "Starts synchronization of message processing based on message originator", nodeDetails = "This node should be used together with \"synchronization end\" node. \n This node will put messages into queue based on message originator id. \n" + "Subsequent messages will not be processed until the previous message processing is completed or timeout event occurs.\n" + "Size of the queue per originator and timeout values are configurable on a system level", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbNodeEmptyConfig") -@Deprecated public class TbSynchronizationBeginNode implements TbNode { + private EmptyNodeConfiguration config; + @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); - ctx.tellSuccess(msg); + log.trace("Msg enters transaction - [{}][{}]", msg.getId(), msg.getType()); + + TbMsgTransactionData transactionData = new TbMsgTransactionData(msg.getId(), msg.getOriginator()); + TbMsg tbMsg = new TbMsg(msg.getId(), msg.getType(), msg.getOriginator(), msg.getMetaData(), TbMsgDataType.JSON, + msg.getData(), transactionData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); + + ctx.getRuleChainTransactionService().beginTransaction(tbMsg, startMsg -> { + log.trace("Transaction starting...[{}][{}]", startMsg.getId(), startMsg.getType()); + ctx.tellNext(startMsg, SUCCESS); + }, endMsg -> log.trace("Transaction ended successfully...[{}][{}]", endMsg.getId(), endMsg.getType()), + throwable -> { + log.trace("Transaction failed! [{}][{}]", tbMsg.getId(), tbMsg.getType(), throwable); + ctx.tellFailure(tbMsg, throwable); + }); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java index 8b83bff1fa..af3742a0fe 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java @@ -35,25 +35,30 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization end", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", + nodeDescription = "Stops synchronization of message processing based on message originator", nodeDetails = "", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = ("tbNodeEmptyConfig") ) -@Deprecated public class TbSynchronizationEndNode implements TbNode { + private EmptyNodeConfiguration config; + @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); - ctx.tellSuccess(msg); + ctx.getRuleChainTransactionService().endTransaction(msg, + successMsg -> ctx.tellNext(successMsg, SUCCESS), + throwable -> ctx.tellFailure(msg, throwable)); + log.trace("Msg left transaction - [{}][{}]", msg.getId(), msg.getType()); } @Override public void destroy() { + } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java index 84a6b76920..e768f699cd 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java @@ -55,7 +55,7 @@ public abstract class TbAbstractTransformNode implements TbNode { protected void transformSuccess(TbContext ctx, TbMsg msg, TbMsg m) { if (m != null) { - ctx.tellSuccess(m); + ctx.tellNext(m, SUCCESS); } else { ctx.tellNext(msg, FAILURE); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java index 31fd207c7c..40a3e8effd 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.id.EntityId; public class EntitiesAlarmOriginatorIdAsyncLoader { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java index 182a000b59..a0a1c8629f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java @@ -22,7 +22,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.EntityFieldsData; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java index 017bacfd72..3ff25e1e8b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.alarm.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index 61b83d647d..67c5b1cc18 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -25,15 +25,12 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; -import org.mockito.Captor; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -41,35 +38,20 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.alarm.AlarmService; import javax.script.ScriptException; import java.io.IOException; import java.util.concurrent.Callable; -import java.util.function.Consumer; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import static org.mockito.Matchers.any; -import static org.mockito.Mockito.anyLong; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.same; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; -import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_CLEARED_ALARM; -import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_EXISTING_ALARM; -import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_NEW_ALARM; +import static org.mockito.Mockito.*; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.*; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.CRITICAL; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.WARNING; -import static org.thingsboard.server.common.data.alarm.AlarmStatus.ACTIVE_UNACK; -import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_ACK; -import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_UNACK; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.*; @RunWith(MockitoJUnitRunner.class) public class TbAlarmNodeTest { @@ -84,11 +66,6 @@ public class TbAlarmNodeTest { @Mock private ScriptEngine detailsJs; - @Captor - private ArgumentCaptor successCaptor; - @Captor - private ArgumentCaptor> failureCaptor; - private RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); private RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); @@ -122,16 +99,15 @@ public class TbAlarmNodeTest { public void newAlarmCanBeCreated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFuture(null)); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); + doAnswer((Answer) invocationOnMock -> (Alarm) (invocationOnMock.getArguments())[0]).when(alarmService).createOrUpdateAlarm(any(Alarm.class)); node.onMsg(ctx, msg); - verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); - successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -165,7 +141,7 @@ public class TbAlarmNodeTest { public void buildDetailsThrowsException() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFailedFuture(new NotImplementedException("message"))); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -188,7 +164,7 @@ public class TbAlarmNodeTest { public void ifAlarmClearedCreateNew() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); Alarm clearedAlarm = Alarm.builder().status(CLEARED_ACK).build(); @@ -199,8 +175,6 @@ public class TbAlarmNodeTest { node.onMsg(ctx, msg); - verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); - successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -235,7 +209,7 @@ public class TbAlarmNodeTest { public void alarmCanBeUpdated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); @@ -282,7 +256,7 @@ public class TbAlarmNodeTest { public void alarmCanBeCleared() throws ScriptException, IOException { initWithClearAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java index b0f64fb980..27feed93ba 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java @@ -19,6 +19,7 @@ import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; @@ -27,24 +28,17 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.same; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) public class TbJsFilterNodeTest { @@ -64,7 +58,7 @@ public class TbJsFilterNodeTest { @Test public void falseEvaluationDoNotSendMsg() throws TbNodeException, ScriptException { initWithScript(); - TbMsg msg = TbMsg.newMsg("USER", null, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(false)); @@ -77,7 +71,7 @@ public class TbJsFilterNodeTest { public void exceptionInJsThrowsException() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFailedFuture(new ScriptException("error"))); @@ -90,7 +84,7 @@ public class TbJsFilterNodeTest { public void metadataConditionCanBeTrue() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(true)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index 9286a06da6..1e752b55b8 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -28,14 +28,10 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -44,9 +40,7 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) public class TbJsSwitchNodeTest { @@ -71,7 +65,7 @@ public class TbJsSwitchNodeTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5}"; - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeSwitch(msg)).thenReturn(Sets.newHashSet("one", "three")); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java index 70f58c5989..ace02bacd7 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java @@ -30,13 +30,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) @@ -62,7 +62,7 @@ public class TbMsgToEmailNodeTest { metaData.putValue("name", "temp"); metaData.putValue("passed", "5"); metaData.putValue("count", "100"); - TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); emailNode.onMsg(ctx, msg); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index a0a61ae30f..0af1cef8ae 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -31,19 +31,9 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.RuleNodeId; -import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.kv.*; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -112,7 +102,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -137,7 +127,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -162,7 +152,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(null)); @@ -176,7 +166,7 @@ public class TbGetCustomerAttributeNodeTest { @Test public void customerAttributeAddedInMetadata() { CustomerId customerId = new CustomerId(UUIDs.timeBased()); - msg = TbMsg.newMsg( "CUSTOMER", customerId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "CUSTOMER", customerId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); entityAttributeFetched(customerId); } @@ -187,7 +177,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -202,7 +192,7 @@ public class TbGetCustomerAttributeNodeTest { Asset asset = new Asset(); asset.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -217,7 +207,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -244,7 +234,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); + msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -256,7 +246,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(timeseries)); node.onMsg(ctx, msg); - verify(ctx).tellSuccess(msg); + verify(ctx).tellNext(msg, SUCCESS); assertEquals(msg.getMetaData().getValue("tempo"), "highest"); } @@ -268,7 +258,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(attributes)); node.onMsg(ctx, msg); - verify(ctx).tellSuccess(msg); + verify(ctx).tellNext(msg, SUCCESS); assertEquals(msg.getMetaData().getValue("tempo"), "high"); } } \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java index fa845c4a18..4c1edce8f6 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java @@ -36,7 +36,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; @@ -92,7 +91,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(),eq( assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -120,7 +119,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -147,7 +146,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(null)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index a00e97cf1c..aa9ad8b76f 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -27,14 +27,10 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.rule.engine.api.TbNodeConfiguration; -import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -42,10 +38,7 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @RunWith(MockitoJUnitRunner.class) @@ -69,15 +62,15 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON,rawJson, ruleChainId, ruleNodeId); - TbMsg transformedMsg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{new}", ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg transformedMsg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{new}", ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFuture(transformedMsg)); node.onMsg(ctx, msg); verify(ctx).getDbCallbackExecutor(); ArgumentCaptor captor = ArgumentCaptor.forClass(TbMsg.class); - verify(ctx).tellSuccess(captor.capture()); + verify(ctx).tellNext(captor.capture(), eq(SUCCESS)); TbMsg actualMsg = captor.getValue(); assertEquals(transformedMsg, actualMsg); } @@ -91,7 +84,7 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFailedFuture(new IllegalStateException("error"))); diff --git a/tools/pom.xml b/tools/pom.xml index a3018e40ce..c8058d1bd7 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -54,7 +54,7 @@ org.apache.cassandra cassandra-all - 3.11.6 + 3.11.4 com.datastax.cassandra diff --git a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java index ec596db10d..6dbca324e1 100644 --- a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java +++ b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.queue.kafka"}) +@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.kafka"}) public class ThingsboardCoapTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c3dcb76508..9b3572c7d1 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -41,143 +41,24 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus - kafka: - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" - topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" - aws_sqs: - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - queue-properties: - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - pubsub: - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - queue-properties: - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - rabbitmq: - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - partitions: - hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" - virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" +kafka: + enabled: true + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" - max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" - request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - core: - topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" - poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - js: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: - - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" - poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" - poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" - pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - transport: - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - -service: - type: "${TB_SERVICE_TYPE:tb-transport}" - # Unique id for this service (autogenerated if empty) - id: "${TB_SERVICE_ID:}" - tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" diff --git a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java index f44af2105e..39dc0677e8 100644 --- a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java +++ b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java @@ -24,7 +24,7 @@ import java.util.Arrays; @SpringBootApplication @EnableAsync -@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.queue.kafka"}) +@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.kafka"}) public class ThingsboardHttpTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index baac6e977f..4097bce831 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -42,143 +42,24 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus - kafka: - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" - topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" - aws_sqs: - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - queue-properties: - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - pubsub: - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - queue-properties: - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - rabbitmq: - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - partitions: - hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" - virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" +kafka: + enabled: true + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" - max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" - request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - core: - topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" - poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - js: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: - - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" - poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" - poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" - pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - transport: - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - -service: - type: "${TB_SERVICE_TYPE:tb-transport}" - # Unique id for this service (autogenerated if empty) - id: "${TB_SERVICE_ID:}" - tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" diff --git a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java index cd7225ad1f..ce95ee6f1f 100644 --- a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java +++ b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.queue"}) +@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.kafka"}) public class ThingsboardMqttTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e920891d66..d13a3d071e 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -17,20 +17,10 @@ spring.main.web-environment: false spring.main.web-application-type: none -# Zookeeper connection parameters. Used for service discovery. -zk: - # Enable/disable zookeeper discovery service. - enabled: "${ZOOKEEPER_ENABLED:false}" - # Zookeeper connect string - url: "${ZOOKEEPER_URL:localhost:2181}" - # Zookeeper retry interval in milliseconds - retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" - # Zookeeper connection timeout in milliseconds - connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" - # Zookeeper session timeout in milliseconds - session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" - # Name of the directory in zookeeper 'filesystem' - zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" +# Clustering properties +cluster: + # Unique id for this node (autogenerated if empty) + node_id: "${CLUSTER_NODE_ID:}" # MQTT server parameters transport: @@ -72,143 +62,24 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus - kafka: - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" - topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" - aws_sqs: - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - queue-properties: - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - pubsub: - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - queue-properties: - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - rabbitmq: - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - partitions: - hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" - virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" +kafka: + enabled: true + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" - max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" - request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - core: - topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" - poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - js: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: - - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" - poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" - poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" - pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" - submit-strategy: - type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL - # For BATCH only - batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch - processing-strategy: - type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - transport: - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - -service: - type: "${TB_SERVICE_TYPE:tb-transport}" - # Unique id for this service (autogenerated if empty) - id: "${TB_SERVICE_ID:}" - tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" diff --git a/ui-ngx/src/app/core/http/public-api.ts b/ui-ngx/src/app/core/http/public-api.ts index db477c3a8c..761f04989d 100644 --- a/ui-ngx/src/app/core/http/public-api.ts +++ b/ui-ngx/src/app/core/http/public-api.ts @@ -28,7 +28,6 @@ export * from './entity-relation.service'; export * from './entity-view.service'; export * from './event.service'; export * from './http-utils'; -export * from './queue.service'; export * from './rule-chain.service'; export * from './tenant.service'; export * from './user.service'; diff --git a/ui-ngx/src/app/core/http/queue.service.ts b/ui-ngx/src/app/core/http/queue.service.ts deleted file mode 100644 index 42beadb81a..0000000000 --- a/ui-ngx/src/app/core/http/queue.service.ts +++ /dev/null @@ -1,36 +0,0 @@ -/// -/// Copyright © 2016-2020 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { Injectable } from '@angular/core'; -import { HttpClient } from '@angular/common/http'; -import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; -import { Observable } from 'rxjs'; -import { ServiceType } from '@shared/models/queue.models'; - -@Injectable({ - providedIn: 'root' -}) -export class QueueService { - - constructor( - private http: HttpClient - ) { } - - public getTenantQueuesByServiceType(serviceType: ServiceType, config?: RequestConfig): Observable> { - return this.http.get>(`/api/tenant/queues?serviceType=${serviceType}`, - defaultHttpOptionsFromConfig(config)); - } -} diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html index 7d0ccd5056..48c57184ab 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html +++ b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.html @@ -56,16 +56,6 @@ -
- -
{{ 'tenant.isolated-tb-core' | translate }}
-
{{'tenant.isolated-tb-core-details' | translate}}
-
- -
{{ 'tenant.isolated-tb-rule-engine' | translate }}
-
{{'tenant.isolated-tb-rule-engine-details' | translate}}
-
-
diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss deleted file mode 100644 index b05c6324fa..0000000000 --- a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.scss +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -:host ::ng-deep { - .mat-checkbox.hinted-checkbox { - .mat-checkbox-inner-container { - margin-top: 4px; - } - } -} diff --git a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts index f624a5d5cf..c8317a20c5 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts +++ b/ui-ngx/src/app/modules/home/pages/tenant/tenant.component.ts @@ -26,8 +26,7 @@ import { EntityTableConfig } from '@home/models/entity/entities-table-config.mod @Component({ selector: 'tb-tenant', - templateUrl: './tenant.component.html', - styleUrls: ['./tenant.component.scss'] + templateUrl: './tenant.component.html' }) export class TenantComponent extends ContactBasedComponent { @@ -51,8 +50,6 @@ export class TenantComponent extends ContactBasedComponent { return this.fb.group( { title: [entity ? entity.title : '', [Validators.required]], - isolatedTbCore: [entity ? entity.isolatedTbCore : false, []], - isolatedTbRuleEngine: [entity ? entity.isolatedTbRuleEngine : false, []], additionalInfo: this.fb.group( { description: [entity && entity.additionalInfo ? entity.additionalInfo.description : ''] @@ -64,8 +61,6 @@ export class TenantComponent extends ContactBasedComponent { updateEntityForm(entity: Tenant) { this.entityForm.patchValue({title: entity.title}); - this.entityForm.patchValue({isolatedTbCore: entity.isolatedTbCore}); - this.entityForm.patchValue({isolatedTbRuleEngine: entity.isolatedTbRuleEngine}); this.entityForm.patchValue({additionalInfo: {description: entity.additionalInfo ? entity.additionalInfo.description : ''}}); } diff --git a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html deleted file mode 100644 index ccdf4a76ec..0000000000 --- a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.html +++ /dev/null @@ -1,43 +0,0 @@ - - - {{ 'queue.name' | translate }} - - - - - - - - - {{ 'queue.name_required' | translate }} - - diff --git a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts b/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts deleted file mode 100644 index 4e008d5c6c..0000000000 --- a/ui-ngx/src/app/shared/components/queue/queue-type-list.component.ts +++ /dev/null @@ -1,177 +0,0 @@ -/// -/// Copyright © 2016-2020 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnDestroy, OnInit, ViewChild } from '@angular/core'; -import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; -import { Observable } from 'rxjs'; -import { map, mergeMap, publishReplay, refCount, tap } from 'rxjs/operators'; -import { Store } from '@ngrx/store'; -import { AppState } from '@app/core/core.state'; -import { TranslateService } from '@ngx-translate/core'; -import { coerceBooleanProperty } from '@angular/cdk/coercion'; -import { QueueService } from '@core/http/queue.service'; -import { ServiceType } from '@shared/models/queue.models'; - -@Component({ - selector: 'tb-queue-type-list', - templateUrl: './queue-type-list.component.html', - styleUrls: [], - providers: [{ - provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => QueueTypeListComponent), - multi: true - }] -}) -export class QueueTypeListComponent implements ControlValueAccessor, OnInit, AfterViewInit, OnDestroy { - - queueFormGroup: FormGroup; - - modelValue: string | null; - - private requiredValue: boolean; - get required(): boolean { - return this.requiredValue; - } - @Input() - set required(value: boolean) { - this.requiredValue = coerceBooleanProperty(value); - } - - @Input() - disabled: boolean; - - @Input() - queueType: ServiceType; - - @ViewChild('queueInput', {static: true}) queueInput: ElementRef; - - filteredQueues: Observable>; - - queues: Observable>; - - searchText = ''; - - private dirty = false; - - private propagateChange = (v: any) => { }; - - constructor(private store: Store, - public translate: TranslateService, - private queueService: QueueService, - private fb: FormBuilder) { - this.queueFormGroup = this.fb.group({ - queue: [null] - }); - } - - registerOnChange(fn: any): void { - this.propagateChange = fn; - } - - registerOnTouched(fn: any): void { - } - - ngOnInit() { - this.filteredQueues = this.queueFormGroup.get('queue').valueChanges - .pipe( - tap(value => { - this.updateView(value); - }), - map(value => value ? value : ''), - mergeMap(queue => this.fetchQueues(queue) ) - ); - } - - ngAfterViewInit(): void { - } - - ngOnDestroy(): void { - } - - setDisabledState(isDisabled: boolean): void { - this.disabled = isDisabled; - if (this.disabled) { - this.queueFormGroup.disable({emitEvent: false}); - } else { - this.queueFormGroup.enable({emitEvent: false}); - } - } - - writeValue(value: string | null): void { - this.searchText = ''; - this.modelValue = value; - this.queueFormGroup.get('queue').patchValue(value, {emitEvent: false}); - this.dirty = true; - } - - onFocus() { - if (this.dirty) { - this.queueFormGroup.get('queue').updateValueAndValidity({onlySelf: true, emitEvent: true}); - this.dirty = false; - } - } - - updateView(value: string | null) { - if (this.modelValue !== value) { - this.modelValue = value; - this.propagateChange(this.modelValue); - } - } - - displayQueueFn(queue?: string): string | undefined { - return queue ? queue : undefined; - } - - fetchQueues(searchText?: string): Observable> { - this.searchText = searchText; - return this.getQueues().pipe( - map(queues => { - const result = queues.filter( queue => { - return searchText ? queue.toUpperCase().startsWith(searchText.toUpperCase()) : true; - }); - if (result.length) { - if (searchText && searchText.length && result.indexOf(searchText) === -1) { - result.push(searchText); - } - result.sort(); - } else if (searchText && searchText.length) { - result.push(searchText); - } - return result; - }) - ); - } - - getQueues(): Observable> { - if (!this.queues) { - this.queues = this.queueService. - getTenantQueuesByServiceType(this.queueType, {ignoreLoading: true}).pipe( - publishReplay(1), - refCount() - ); - } - return this.queues; - } - - clear() { - this.queueFormGroup.get('queue').patchValue(null, {emitEvent: true}); - setTimeout(() => { - this.queueInput.nativeElement.blur(); - this.queueInput.nativeElement.focus(); - }, 0); - } - -} diff --git a/ui-ngx/src/app/shared/models/public-api.ts b/ui-ngx/src/app/shared/models/public-api.ts index cbc030a4fa..856c994943 100644 --- a/ui-ngx/src/app/shared/models/public-api.ts +++ b/ui-ngx/src/app/shared/models/public-api.ts @@ -36,7 +36,6 @@ export * from './error.models'; export * from './event.models'; export * from './login.models'; export * from './material.models'; -export * from './queue.models'; export * from './relation.models'; export * from './rule-chain.models'; export * from './rule-node.models'; diff --git a/ui-ngx/src/app/shared/models/queue.models.ts b/ui-ngx/src/app/shared/models/queue.models.ts deleted file mode 100644 index ff8f386dbe..0000000000 --- a/ui-ngx/src/app/shared/models/queue.models.ts +++ /dev/null @@ -1,22 +0,0 @@ -/// -/// Copyright © 2016-2020 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -export enum ServiceType { - TB_CORE = 'TB_CORE', - TB_RULE_ENGINE = 'TB_RULE_ENGINE', - TB_TRANSPORT = 'TB_TRANSPORT', - JS_EXECUTOR = 'JS_EXECUTOR' -} diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 4f2e01ea81..266981db1f 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -21,7 +21,5 @@ import {TenantId} from './id/tenant-id'; export interface Tenant extends ContactBased { title: string; region: string; - isolatedTbCore: boolean; - isolatedTbRuleEngine: boolean; additionalInfo?: any; } diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index e94c8a6641..5008f10459 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -129,7 +129,6 @@ import { JsonObjectEditDialogComponent } from '@shared/components/dialog/json-ob import { HistorySelectorComponent } from './components/time/history-selector/history-selector.component'; import { TbTemplatePipe } from '@shared/pipe/template.pipe'; import { EntityGatewaySelectComponent } from '@shared/components/entity/entity-gateway-select.component'; -import { QueueTypeListComponent } from '@shared/components/queue/queue-type-list.component'; @NgModule({ providers: [ @@ -180,7 +179,6 @@ import { QueueTypeListComponent } from '@shared/components/queue/queue-type-list EntityKeysListComponent, EntityListSelectComponent, EntityTypeListComponent, - QueueTypeListComponent, RelationTypeAutocompleteComponent, SocialSharePanelComponent, JsonObjectEditComponent, @@ -299,7 +297,6 @@ import { QueueTypeListComponent } from '@shared/components/queue/queue-type-list EntityKeysListComponent, EntityListSelectComponent, EntityTypeListComponent, - QueueTypeListComponent, RelationTypeAutocompleteComponent, SocialSharePanelComponent, JsonObjectEditComponent, diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 2a179b3798..3112cba1af 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1514,11 +1514,6 @@ "help": "Help", "reset-debug-mode": "Reset debug mode in all nodes" }, - "queue": { - "select_name": "Select queue name", - "name": "Queue Name", - "name_required": "Queue name is required!" - }, "tenant": { "tenant": "Tenant", "tenants": "Tenants", @@ -1546,12 +1541,7 @@ "no-tenants-matching": "No tenants matching '{{entity}}' were found.", "tenant-required": "Tenant is required", "search": "Search tenants", - "selected-tenants": "{ count, plural, 1 {1 tenant} other {# tenants} } selected", - "tenant-required": "Tenant is required", - "isolated-tb-core": "Processing in isolated ThingsBoard Core container", - "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", - "isolated-tb-core-details": "Requires separate microservice(s) per isolated Tenant", - "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" + "selected-tenants": "{ count, plural, 1 {1 tenant} other {# tenants} } selected" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", diff --git a/ui-ngx/src/assets/locale/locale.constant-ru_RU.json b/ui-ngx/src/assets/locale/locale.constant-ru_RU.json index 0d6c9abdc9..514cd9a8e3 100644 --- a/ui-ngx/src/assets/locale/locale.constant-ru_RU.json +++ b/ui-ngx/src/assets/locale/locale.constant-ru_RU.json @@ -1404,11 +1404,6 @@ "help": "Помощь", "reset-debug-mode": "Сбросить режим отладки во всех правилах" }, - "queue": { - "select_name": "Выберите имя для Queue", - "name": "Имя для Queue", - "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" - }, "tenant": { "tenant": "Владелец", "tenants": "Владельцы", diff --git a/ui-ngx/src/assets/locale/locale.constant-uk_UA.json b/ui-ngx/src/assets/locale/locale.constant-uk_UA.json index fa0da05e0a..92e8dc4a99 100644 --- a/ui-ngx/src/assets/locale/locale.constant-uk_UA.json +++ b/ui-ngx/src/assets/locale/locale.constant-uk_UA.json @@ -1970,11 +1970,6 @@ "no-timezones-matching": "Не знайдено жодних часових поясів, які відповідають '{{timezone}}'.", "timezone-required": "Необхідно вказати часовий пояс." }, - "queue": { - "select_name": "Виберіть ім'я для Queue", - "name": "Iм'я для Queue", - "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" - }, "tenant": { "tenant": "Власник", "tenants": "Власники", diff --git a/ui/package-lock.json b/ui/package-lock.json index bd0e0cc260..6a9997cbf0 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1095,12 +1095,12 @@ "@flowjs/ng-flow": { "version": "2.7.8", "resolved": "https://registry.npmjs.org/@flowjs/ng-flow/-/ng-flow-2.7.8.tgz", - "integrity": "sha1-HZ+dH4Ks2lNgMowxW6z9YNv9mBk=" + "integrity": "sha512-zO6jNvz41oMOJj9+1N+vLT0ytitbCtuGABJQRzQDOPXyRMmlSXfJ7om5oYOztyUFrr4jDpE4QFPt+r2/RFceCg==" }, "@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", - "integrity": "sha1-UkryQNGjYFJ7cwR17PoTRKpUDd4=", + "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==", "dev": true, "requires": { "call-me-maybe": "^1.0.1", @@ -1424,7 +1424,7 @@ "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha1-+PLIh60Qv2f2NPAFtph/7TF5qsg=" + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, "accepts": { "version": "1.3.7", @@ -1534,7 +1534,7 @@ "angular-carousel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/angular-carousel/-/angular-carousel-1.1.0.tgz", - "integrity": "sha1-PmlA5ovRio85L8Qx2XGSrDSIMdE=" + "integrity": "sha512-UiLMgT7Ueqk4xpliF1gWt4dYKXezdJA1jyZPNsUWkOGO/dwLuKi284h3BgWl4CnaH7kEBw8L2gsBOyqbYaumNQ==" }, "angular-cookies": { "version": "1.5.8", @@ -1555,7 +1555,7 @@ } }, "angular-fullscreen": { - "version": "git://github.com/fabiobiondi/angular-fullscreen.git#8217174565761d3566807bc60a73b5ca015b8cb6", + "version": "git://github.com/fabiobiondi/angular-fullscreen.git#119b7fbac911d154fd56ace38ebe3432475e8a20", "from": "git://github.com/fabiobiondi/angular-fullscreen.git#master" }, "angular-gridster": { @@ -1629,7 +1629,7 @@ "angular-translate": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate/-/angular-translate-2.18.1.tgz", - "integrity": "sha1-sp7Q0vm6xEB156rTKEFmxZ4VB5E=", + "integrity": "sha512-Mw0kFBqsv5j8ItL9IhRZunIlVmIRW6iFsiTmRs9wGr2QTt8z4rehYlWyHos8qnXc/kyOYJiW50iH50CSNHGB9A==", "requires": { "angular": ">=1.2.26 <=1.7" } @@ -1637,7 +1637,7 @@ "angular-translate-handler-log": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-handler-log/-/angular-translate-handler-log-2.18.1.tgz", - "integrity": "sha1-icu1mCeALYb4EVJ1+/iNbYiWsNQ=", + "integrity": "sha512-TyKzCW4GubNazwCgLpCVXd2212CWdZOckf+aL5+gLuThPhVpOvlg18RSmz8MNPto3kwCcCw3LzShlZ6RX/MQRA==", "requires": { "angular-translate": "~2.18.1" } @@ -1645,7 +1645,7 @@ "angular-translate-interpolation-messageformat": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-interpolation-messageformat/-/angular-translate-interpolation-messageformat-2.18.1.tgz", - "integrity": "sha1-FsUq4MYcJA8PJBZKBSGUPPi6QI4=", + "integrity": "sha512-SlmyxLB/UUy7FWoGx5QJHrhq8fUu/xzCR0h/ngexOtXZopQjs1vm+TrFZ69d4c/LI7C91sfP4mq4ES29o1xCxA==", "requires": { "angular-translate": "~2.18.1", "messageformat": "~1.0.2" @@ -1654,7 +1654,7 @@ "angular-translate-loader-static-files": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-loader-static-files/-/angular-translate-loader-static-files-2.18.1.tgz", - "integrity": "sha1-rQw8iDsYsIm9uNsCu9Nm2QP4V8w=", + "integrity": "sha512-5MuyzAROfc493kjLjKlLGLBzXiRmZIFbcWZGutDRxW5SRXSpwrH0u0hh0ENNnUyUQbe2vUspHNPIuZqlq8qIhw==", "requires": { "angular-translate": "~2.18.1" } @@ -1662,7 +1662,7 @@ "angular-translate-storage-cookie": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-cookie/-/angular-translate-storage-cookie-2.18.1.tgz", - "integrity": "sha1-j8vaspb6gkkOALQorxp0ahf0QVY=", + "integrity": "sha512-wiMaF/0OGN/3ilaYunfsqdLNpfGZEJK0fj4zT8yjD3XPq7Q9kM88xZ4XJiWKgodZShBljGCRzqgQbKMF7d1MLw==", "requires": { "angular-cookies": ">=1.2.26 <1.8", "angular-translate": "~2.18.1" @@ -1671,7 +1671,7 @@ "angular-translate-storage-local": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-local/-/angular-translate-storage-local-2.18.1.tgz", - "integrity": "sha1-lHQP5NgBq3gpopofBeHDkFTIcwM=", + "integrity": "sha512-zPxcbIJ8tdWXtWNKLtaswynKid0w5le6WPMwiLWhgKPnyzOp/y5WLBW+JEfnZnkGE24yOGhJ6jVPgRNzelLgzg==", "requires": { "angular-translate": "~2.18.1", "angular-translate-storage-cookie": "~2.18.1" @@ -1750,7 +1750,7 @@ "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha1-aALmJk79GMeQobDVF/DyYnvyyUo=", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "dev": true }, "are-we-there-yet": { @@ -1766,7 +1766,7 @@ "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "dev": true, "requires": { "sprintf-js": "~1.0.2" @@ -1781,7 +1781,7 @@ "arr-flatten": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", - "integrity": "sha1-NgSLv/TntH4TZkQxbJlmnqWukfE=", + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", "dev": true }, "arr-union": { @@ -1893,7 +1893,7 @@ }, "util": { "version": "0.10.3", - "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1958,7 +1958,7 @@ "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", - "integrity": "sha1-bZUX654DDSQ2ZmZR6GvZ9vE1M8k=", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", "dev": true }, "attr-accept": { @@ -2102,7 +2102,7 @@ "base": { "version": "0.11.2", "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", - "integrity": "sha1-e95c7RRbbVUakNuH+DxVi060io8=", + "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", "dev": true, "requires": { "cache-base": "^1.0.1", @@ -2126,7 +2126,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2135,7 +2135,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2144,7 +2144,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -2290,7 +2290,7 @@ "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2514,7 +2514,7 @@ "cache-base": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", - "integrity": "sha1-Cn9GQWgxyLZi7jb+TnxZ129marI=", + "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", "dev": true, "requires": { "collection-visit": "^1.0.0", @@ -2545,7 +2545,7 @@ "dependencies": { "callsites": { "version": "2.0.0", - "resolved": "http://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", "integrity": "sha1-BuuE8A7qQT2oav/vrL/7Ngk7PFA=", "dev": true } @@ -2722,7 +2722,7 @@ "class-utils": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", - "integrity": "sha1-+TNprouafOAv1B+q0MqDAzGQxGM=", + "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", "dev": true, "requires": { "arr-union": "^3.1.0", @@ -3049,7 +3049,7 @@ "concat-stream": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha1-kEvfGUzTEi/Gdcd/xKw9T/D9GjQ=", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", "dev": true, "requires": { "buffer-from": "^1.0.0", @@ -3109,7 +3109,7 @@ "content-type": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha1-4TjMdeBAxyexlm/l5fjJruJW/js=", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", "dev": true }, "convert-source-map": { @@ -3136,7 +3136,7 @@ "copy-concurrently": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz", - "integrity": "sha1-kilzmMrjSTf8r9bsgTnBgFHwteA=", + "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==", "dev": true, "requires": { "aproba": "^1.1.1", @@ -3355,7 +3355,7 @@ "create-react-class": { "version": "15.6.3", "resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.6.3.tgz", - "integrity": "sha1-LXMjf7P5cK5uvgEanmb0bbyoADY=", + "integrity": "sha512-M+/3Q6E6DLO6Yx3OwrWjwHBnvfXXYA7W+dFjt/ZDBemHO1DDZhsalX/NUtnTYclN6GfnBDRh4qRHjcDHmlJBJg==", "requires": { "fbjs": "^0.8.9", "loose-envify": "^1.3.1", @@ -3483,7 +3483,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "dev": true, "requires": { "ms": "2.0.0" @@ -3561,7 +3561,7 @@ "define-property": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", - "integrity": "sha1-1Flono1lS6d+AqgX+HENcCyxbp0=", + "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", "dev": true, "requires": { "is-descriptor": "^1.0.2", @@ -3571,7 +3571,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3580,7 +3580,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3589,7 +3589,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -3652,7 +3652,7 @@ "delegate": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", - "integrity": "sha1-tmtxwxWFIuirV0T3INjKDCr1kWY=" + "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" }, "delegates": { "version": "1.0.0", @@ -3808,7 +3808,7 @@ "domain-browser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", - "integrity": "sha1-PTH1AZGmdJ3RN1p/Ui6CPULlTto=", + "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", "dev": true }, "domelementtype": { @@ -3972,7 +3972,7 @@ "errno": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", - "integrity": "sha1-RoTXF3mtOa8Xfj8AeZb3xnyFJhg=", + "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", "dev": true, "requires": { "prr": "~1.0.1" @@ -4491,7 +4491,7 @@ "esrecurse": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", - "integrity": "sha1-AHo7n9vCs7uH5IeeoZyS/b05Qs8=", + "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", "dev": true, "requires": { "estraverse": "^4.1.0" @@ -4706,7 +4706,7 @@ "external-editor": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz", - "integrity": "sha1-BFURz9jRM/OEZnPRBHwVTiFK09U=", + "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==", "requires": { "chardet": "^0.4.0", "iconv-lite": "^0.4.17", @@ -5134,7 +5134,7 @@ "fs-readdir-recursive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz", - "integrity": "sha1-4y/AMKLM7kSmtTcTCNpUvgs5fSc=", + "integrity": "sha512-GNanXlVr2pf02+sPN40XN8HG+ePaNcvM0q5mZBd668Obwb0yD5GiUbZOFgwn8kGMY6I3mdyDJzieUy3PTYyTRA==", "dev": true }, "fs-write-stream-atomic": { @@ -5175,7 +5175,8 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -5196,12 +5197,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5216,17 +5219,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -5343,7 +5349,8 @@ "inherits": { "version": "2.0.4", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -5355,6 +5362,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5369,6 +5377,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5376,12 +5385,14 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.9.0", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5400,6 +5411,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -5489,7 +5501,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -5501,6 +5514,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -5586,7 +5600,8 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -5622,6 +5637,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5641,6 +5657,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5684,12 +5701,14 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true } } }, @@ -5708,7 +5727,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha1-pWiZ0+o8m6uHS7l3O3xe3pL0iV0=", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", "dev": true }, "functional-red-black-tree": { @@ -6662,7 +6681,7 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha1-7uJfVtscnsYIXgwid4CD9Zar+Sc=" + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" }, "inline-style-prefixer": { "version": "2.0.5", @@ -6712,7 +6731,7 @@ "invariant": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", - "integrity": "sha1-YQ88ksk1nOHbYW5TgAjSP/NRWOY=", + "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", "dev": true, "requires": { "loose-envify": "^1.0.0" @@ -6814,7 +6833,7 @@ "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha1-76ouqdqg16suoTqXsritUf776L4=", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", "dev": true }, "is-callable": { @@ -6858,7 +6877,7 @@ "is-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", - "integrity": "sha1-Nm2CQN3kh8pRgjsaufB6EKeCUco=", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", "dev": true, "requires": { "is-accessor-descriptor": "^0.1.6", @@ -6869,7 +6888,7 @@ "kind-of": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", - "integrity": "sha1-cpyR4thXt6QZofmqZWhcTDP1hF0=", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", "dev": true } } @@ -6977,7 +6996,7 @@ "is-plain-object": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", - "integrity": "sha1-LBY7P6+xtgbZ0Xko8FwqHDjgdnc=", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", "dev": true, "requires": { "isobject": "^3.0.1" @@ -7044,7 +7063,7 @@ "is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha1-0YUOuXkezRjmGCzhKjDzlmNLsZ0=", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", "dev": true }, "is-word-character": { @@ -7184,7 +7203,7 @@ "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "integrity": "sha1-u4Z8+zRQ5pEHwTHRxRS6s9yLyqk=", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", "dev": true }, "json-schema": { @@ -7201,7 +7220,7 @@ "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA=", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, "json-stable-stringify-without-jsonify": { @@ -7441,7 +7460,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -7974,7 +7993,7 @@ "messageformat-parser": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-1.1.0.tgz", - "integrity": "sha1-E7oiUKdrvejg/KDbs0dflcWUqQo=" + "integrity": "sha512-Hwem6G3MsKDLS1FtBRGIs8T50P1Q00r3srS6QJePCFbad9fq0nYxwf3rnU2BreApRGhmpKMV7oZI06Sy1c9TPA==" }, "methods": { "version": "1.1.2", @@ -8016,7 +8035,7 @@ "mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha1-Ms2eXGRVO9WNGaVor0Uqz/BJgbE=", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", "dev": true }, "mime-db": { @@ -8037,7 +8056,7 @@ "mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha1-ggyGo5M0ZA6ZUWkovQP8qIBX0CI=" + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" }, "min-document": { "version": "2.19.0", @@ -8094,7 +8113,7 @@ "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "requires": { "brace-expansion": "^1.1.7" } @@ -8189,7 +8208,7 @@ "is-extendable": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha1-p0cPnkJnM9gb2B4RVSZOOjUHyrQ=", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", "dev": true, "requires": { "is-plain-object": "^2.0.4" @@ -8267,7 +8286,7 @@ "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", - "integrity": "sha1-uHqKpPwN6P5r6IiVs4mD/yZb0Rk=", + "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", "dev": true, "requires": { "arr-diff": "^4.0.0", @@ -8425,7 +8444,7 @@ } }, "ngFlowchart": { - "version": "git://github.com/thingsboard/ngFlowchart.git#1343a7478961f68280d81f0ecda4e722a2068e0f", + "version": "git://github.com/thingsboard/ngFlowchart.git#b941e4ed38c226019890b7b0802b71c2b147f0e0", "from": "git://github.com/thingsboard/ngFlowchart.git#master" }, "ngclipboard": { @@ -8485,7 +8504,7 @@ "no-case": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/no-case/-/no-case-2.3.2.tgz", - "integrity": "sha1-YLgTOWvjmz8SiKTB7V0efSi0ZKw=", + "integrity": "sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==", "dev": true, "requires": { "lower-case": "^1.1.1" @@ -8504,7 +8523,7 @@ "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", - "integrity": "sha1-mA9vcthSEaU0fGsrwYxbhMPrR+8=", + "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", "requires": { "encoding": "^0.1.11", "is-stream": "^1.0.1" @@ -8705,7 +8724,7 @@ "npmlog": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha1-CKfyqL9zRgR3mp76StXMcXq7lUs=", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", "dev": true, "requires": { "are-we-there-yet": "~1.1.2", @@ -9022,7 +9041,7 @@ "osenv": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", - "integrity": "sha1-hc36+uso6Gd/QW4odZK18/SepBA=", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", "requires": { "os-homedir": "^1.0.0", "os-tmpdir": "^1.0.0" @@ -9450,7 +9469,7 @@ "postcss-loader": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/postcss-loader/-/postcss-loader-3.0.0.tgz", - "integrity": "sha1-a5eUPkfHLYRfqeA/Jzdz1OjdbC0=", + "integrity": "sha512-cLWoDEY5OwHcAjDnkyRQzAXfs2jrKjXpO/HQFcc5b5u/r7aa471wdmChmwfnv7x2u840iat/wi0lQ5nbRgSkUA==", "dev": true, "requires": { "loader-utils": "^1.1.0", @@ -9694,7 +9713,7 @@ "private": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", - "integrity": "sha1-I4Hts2ifelPWUxkAYPz4ItLzaP8=", + "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==", "dev": true }, "process": { @@ -9717,7 +9736,7 @@ "promise": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", - "integrity": "sha1-BktyYCsY+Q8pGSuLG8QY/9Hr078=", + "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", "requires": { "asap": "~2.0.3" } @@ -9797,7 +9816,7 @@ "pumpify": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", - "integrity": "sha1-NlE74karJ1cLGjdKXOJ4v9dDcM4=", + "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", "dev": true, "requires": { "duplexify": "^3.6.0", @@ -9962,7 +9981,7 @@ "rc-menu": { "version": "5.1.4", "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-5.1.4.tgz", - "integrity": "sha1-5d8I/ouDPoFGkTX/E7MKuPIf88Y=", + "integrity": "sha512-ZUkUNda70GtTXcQDiO3rSDdk3sgIwDwzPUm5dVM8nRH/j84qv0BVBkIUwIBu8+s+G3G9lWLurRqh22dCqZPeOA==", "requires": { "babel-runtime": "6.x", "classnames": "2.x", @@ -9993,7 +10012,7 @@ "rc-trigger": { "version": "1.11.5", "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-1.11.5.tgz", - "integrity": "sha1-+I+fhODnn44O8cjRv4rCIItxViA=", + "integrity": "sha512-MBuUPw1nFzA4K7jQOwb7uvFaZFjXGd00EofUYiZ+l/fgKVq8wnLC0lkv36kwqM7vfKyftRo2sh7cWVpdPuNnnw==", "requires": { "babel-runtime": "6.x", "create-react-class": "15.x", @@ -10152,7 +10171,7 @@ "react-transition-group": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-1.2.1.tgz", - "integrity": "sha1-4R9yslf5IbITIpp3TfRmEjRsfKY=", + "integrity": "sha512-CWaL3laCmgAFdxdKbhhps+c0HRGF4c+hdM4H23+FI1QBNUyx/AMeIJGWorehPNSaKnQNOAxL7PQmqMu78CDj3Q==", "requires": { "chain-function": "^1.0.0", "dom-helpers": "^3.2.0", @@ -10164,7 +10183,7 @@ "reactcss": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/reactcss/-/reactcss-1.2.3.tgz", - "integrity": "sha1-wAATh15Vexzw39mjaKHD2rO1SN0=", + "integrity": "sha512-KiwVUcFu1RErkI97ywr8nvx8dNOpT03rbnma0SSalTYjkrPYaEajR4a/MRt6DZ46K6arDRbWMNHF+xH7G7n/8A==", "requires": { "lodash": "^4.0.1" } @@ -10347,7 +10366,7 @@ "regenerator-runtime": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", - "integrity": "sha1-vgWtf5v30i4Fb5cmzuUBf78Z4uk=" + "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" }, "regenerator-transform": { "version": "0.14.1", @@ -10361,7 +10380,7 @@ "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", - "integrity": "sha1-H07OJ+ALC2XgJHpoEOaoXYOldSw=", + "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", "dev": true, "requires": { "extend-shallow": "^3.0.2", @@ -10666,7 +10685,7 @@ "ret": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", - "integrity": "sha1-uKSCXVvbH8P29Twrwz+BOIaBx7w=", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", "dev": true }, "retry": { @@ -10770,7 +10789,7 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "sass-graph": { "version": "2.2.4", @@ -11104,7 +11123,7 @@ "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", - "integrity": "sha1-ZJIufFZbDhQgS6GqfWlkJ40lGC0=", + "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", "dev": true, "requires": { "base": "^0.11.1", @@ -11140,7 +11159,7 @@ "snapdragon-node": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", - "integrity": "sha1-bBdfhv8UvbByRWPo88GwIaKGhTs=", + "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", "dev": true, "requires": { "define-property": "^1.0.0", @@ -11160,7 +11179,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11169,7 +11188,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11178,7 +11197,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -11191,7 +11210,7 @@ "snapdragon-util": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", - "integrity": "sha1-+VZHlIbyrNeXAGk/b3uAXkWrVuI=", + "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", "dev": true, "requires": { "kind-of": "^3.2.0" @@ -11211,7 +11230,7 @@ "sockjs": { "version": "0.3.19", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.19.tgz", - "integrity": "sha1-2Xa76ACve9IK4IWY1YI5NQiZPA0=", + "integrity": "sha512-V48klKZl8T6MzatbLlzzRNhMepEys9Y4oGFpypBFFn1gLI/QQ9HtLLyWJNbPlwGLelOVOEijUbTTJeLLI59jLw==", "dev": true, "requires": { "faye-websocket": "^0.10.0", @@ -11335,7 +11354,7 @@ "spdx-expression-parse": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", - "integrity": "sha1-meEZt6XaAOBUkcn6M4t5BII7QdA=", + "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", "dev": true, "requires": { "spdx-exceptions": "^2.1.0", @@ -11429,7 +11448,7 @@ "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", - "integrity": "sha1-fLCd2jqGWFcFxks5pkZgOGguj+I=", + "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", "dev": true, "requires": { "extend-shallow": "^3.0.0" @@ -11569,7 +11588,7 @@ "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha1-q5Pyeo3BPSjKyBXEYhQ6bZASrp4=", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", "requires": { "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^4.0.0" @@ -11613,7 +11632,7 @@ "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "requires": { "safe-buffer": "~5.1.0" } @@ -12298,7 +12317,7 @@ "symbol-observable": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", - "integrity": "sha1-wiaIrtTqs83C3+rLtWFmBWCgCAQ=" + "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==" }, "table": { "version": "5.4.6", @@ -12528,7 +12547,7 @@ "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha1-bTQzWIl2jSGyvNoKonfO07G/rfk=", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", "requires": { "os-tmpdir": "~1.0.2" } @@ -12568,7 +12587,7 @@ "to-regex": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", - "integrity": "sha1-E8/dmzNlUvMLUfM6iuG0Knp1mc4=", + "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", "dev": true, "requires": { "define-property": "^2.0.2", @@ -12770,7 +12789,7 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "dev": true } } @@ -13095,7 +13114,7 @@ "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha1-lMVA4f93KVbiKZUHwBCupsiDjrA=", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", "dev": true, "requires": { "punycode": "^2.1.0" @@ -13157,7 +13176,7 @@ "use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", - "integrity": "sha1-1QyMrHmhn7wg8pEfVuuXP04QBw8=", + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", "dev": true }, "util": { @@ -13922,7 +13941,7 @@ "websocket-extensions": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", - "integrity": "sha1-XS/yKXcAPsaHpLhwc9+7rBRszyk=", + "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==", "dev": true }, "whatwg-fetch": { @@ -14036,7 +14055,7 @@ "ws": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ws/-/ws-1.1.5.tgz", - "integrity": "sha1-y9nm514J/F0skAFfIfDECHXg3VE=", + "integrity": "sha512-o3KqipXNUdS7wpQzBHSe180lBGO60SoK0yVo3CYJgb2MkobuWuBX6dhkYP5ORCLd55y+SaflMOV5fqAB53ux4w==", "requires": { "options": ">=0.0.5", "ultron": "1.0.x" diff --git a/ui/src/app/api/queue.service.js b/ui/src/app/api/queue.service.js deleted file mode 100644 index 0fe93e5fa6..0000000000 --- a/ui/src/app/api/queue.service.js +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export default angular.module('thingsboard.api.queue', []) - .factory('queueService', queueService) - .name; - -/*@ngInject*/ -function queueService($http, $q) { - var service = { - getTenantQueuesByServiceType: getTenantQueuesByServiceType - }; - - return service; - - function getTenantQueuesByServiceType(serviceType, config) { - let deferred = $q.defer(); - let url = '/api/tenant/queues?serviceType=' + serviceType; - - $http.get(url, config).then(function success(data) { - deferred.resolve(data); - }, function fail() { - deferred.reject(); - }); - return deferred.promise; - } -} \ No newline at end of file diff --git a/ui/src/app/components/queue/index.js b/ui/src/app/components/queue/index.js deleted file mode 100644 index 5236603270..0000000000 --- a/ui/src/app/components/queue/index.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import thingsboardApiQueue from '../../api/queue.service'; -import queueTypeList from "./queue-type-list.directive"; - -export default angular.module('thingsboard.queue', [ - thingsboardApiQueue -]) - .directive('tbQueueTypeList', queueTypeList) - .name; diff --git a/ui/src/app/components/queue/queue-type-list.directive.js b/ui/src/app/components/queue/queue-type-list.directive.js deleted file mode 100644 index 6a2f99d280..0000000000 --- a/ui/src/app/components/queue/queue-type-list.directive.js +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import './queue-type-list.scss'; - -/* eslint-disable import/no-unresolved, import/default */ - -import queueTypeListTemplate from './queue-type-list.tpl.html'; - -/* eslint-enable import/no-unresolved, import/default */ - -/*@ngInject*/ -export default function QueueTypeList($compile, $templateCache, $q, $filter, queueService) { - - var linker = function (scope, element, attrs, ngModelCtrl) { - var template = $templateCache.get(queueTypeListTemplate); - element.html(template); - - scope.queues = null; - scope.queue = null; - scope.tbRequired = angular.isDefined(scope.tbRequired) ? scope.tbRequired : false; - scope.queueSearchText = ''; - - scope.fetchQueues = function(searchText) { - var deferred = $q.defer(); - loadQueues().then( - function success(queueArr) { - let result = $filter('filter')(queueArr, {'$': searchText}); - if (result && result.length) { - if (searchText && searchText.length && result.indexOf(searchText) === -1) { - result.push(searchText); - } - result.sort(); - deferred.resolve(result); - } else { - deferred.resolve([searchText]); - } - }, - function fail() { - deferred.reject(); - } - ); - - return deferred.promise; - }; - - scope.updateView = function () { - if (!scope.disabled) { - ngModelCtrl.$setViewValue(scope.queue); - } - }; - - function loadQueues() { - var deferred = $q.defer(); - if (!scope.queues) { - queueService.getTenantQueuesByServiceType(scope.queueType).then( - function success(queueArr) { - scope.queues = queueArr.data; - deferred.resolve(scope.queues); - }, - function fail() { - deferred.reject(); - } - ); - } else { - deferred.resolve(scope.queues); - } - return deferred.promise; - } - - ngModelCtrl.$render = function () { - scope.queue = ngModelCtrl.$viewValue; - }; - - scope.$watch('queue', function (newValue, prevValue) { - if (!angular.equals(newValue, prevValue)) { - scope.updateView(); - } - }); - - scope.$watch('disabled', function () { - scope.updateView(); - }); - - $compile(element.contents())(scope); - }; - - return { - restrict: "E", - require: "^ngModel", - link: linker, - scope: { - theForm: '=?', - tbRequired: '=?', - disabled:'=ngDisabled', - queueType: '=?' - } - }; -} \ No newline at end of file diff --git a/ui/src/app/components/queue/queue-type-list.scss b/ui/src/app/components/queue/queue-type-list.scss deleted file mode 100644 index b8f6f58fb2..0000000000 --- a/ui/src/app/components/queue/queue-type-list.scss +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/ui/src/app/components/queue/queue-type-list.tpl.html b/ui/src/app/components/queue/queue-type-list.tpl.html deleted file mode 100644 index 5fc700026e..0000000000 --- a/ui/src/app/components/queue/queue-type-list.tpl.html +++ /dev/null @@ -1,43 +0,0 @@ - - - -
- {{item}} -
-
-
-
{{'queue.name_required' | translate}}
-
-
\ No newline at end of file diff --git a/ui/src/app/layout/index.js b/ui/src/app/layout/index.js index f3b4d10e4f..a00baa5199 100644 --- a/ui/src/app/layout/index.js +++ b/ui/src/app/layout/index.js @@ -55,7 +55,6 @@ import thingsboardEntityView from '../entity-view'; import thingsboardWidgetLibrary from '../widget'; import thingsboardDashboard from '../dashboard'; import thingsboardRuleChain from '../rulechain'; -import thingsboardQueue from '../components/queue'; import thingsboardJsonForm from '../jsonform'; @@ -87,7 +86,6 @@ export default angular.module('thingsboard.home', [ thingsboardWidgetLibrary, thingsboardDashboard, thingsboardRuleChain, - thingsboardQueue, thingsboardJsonForm, thingsboardApiDevice, thingsboardApiLogin, diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index fa4689fc75..8c7a89580d 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1489,12 +1489,6 @@ "help": "Help", "reset-debug-mode": "Reset debug mode in all nodes" }, - "queue": { - "select_name": "Select queue name", - "name": "Queue Name", - "name_required": "Queue name is required!" - - }, "tenant": { "tenant": "Tenant", "tenants": "Tenants", @@ -1520,11 +1514,7 @@ "idCopiedMessage": "Tenant Id has been copied to clipboard", "select-tenant": "Select tenant", "no-tenants-matching": "No tenants matching '{{entity}}' were found.", - "tenant-required": "Tenant is required", - "isolated-tb-core": "Processing in isolated ThingsBoard Core container", - "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", - "isolated-tb-core-details": "Requires separate microservice(s) per isolated Tenant", - "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" + "tenant-required": "Tenant is required" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", diff --git a/ui/src/app/locale/locale.constant-ru_RU.json b/ui/src/app/locale/locale.constant-ru_RU.json index e93c92124c..f86eb988ff 100644 --- a/ui/src/app/locale/locale.constant-ru_RU.json +++ b/ui/src/app/locale/locale.constant-ru_RU.json @@ -1403,12 +1403,6 @@ "help": "Помощь", "reset-debug-mode": "Сбросить режим отладки во всех правилах" }, - "queue": { - "select_name": "Выберите имя для Queue", - "name": "Имя для Queue", - "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" - - }, "tenant": { "tenant": "Владелец", "tenants": "Владельцы", diff --git a/ui/src/app/locale/locale.constant-uk_UA.json b/ui/src/app/locale/locale.constant-uk_UA.json index c19049bc11..5a0899014d 100644 --- a/ui/src/app/locale/locale.constant-uk_UA.json +++ b/ui/src/app/locale/locale.constant-uk_UA.json @@ -1820,12 +1820,6 @@ "help": "Допомога", "reset-debug-mode": "Вимкнути режим налогодження у всіх правилах" }, - "queue": { - "select_name": "Виберіть ім'я для Queue", - "name": "Iм'я для Queue", - "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" - - }, "scheduler": { "scheduler": "Планувальник", "scheduler-event": "Подія планувальника", diff --git a/ui/src/app/tenant/tenant-fieldset.tpl.html b/ui/src/app/tenant/tenant-fieldset.tpl.html index b75ea30f06..aa7751d479 100644 --- a/ui/src/app/tenant/tenant-fieldset.tpl.html +++ b/ui/src/app/tenant/tenant-fieldset.tpl.html @@ -15,50 +15,32 @@ limitations under the License. --> -{{ - 'tenant.manage-tenant-admins' | translate }} - -{{ 'tenant.delete' - | translate }} - +{{ 'tenant.manage-tenant-admins' | translate }} +{{ 'tenant.delete' | translate }}
- - - tenant.copyId - + + + tenant.copyId +
-
- - - -
-
tenant.title-required
-
-
- - - - - - - - {{'tenant.isolated-tb-core' | translate}}
- {{'tenant.isolated-tb-core-details' | translate}} -
-
- - - {{'tenant.isolated-tb-rule-engine' | translate}}
- {{'tenant.isolated-tb-rule-engine-details' | translate}} -
-
-
+
+ + + +
+
tenant.title-required
+
+
+ + + + + +