diff --git a/.gitignore b/.gitignore index c0628fd2e3..dd48058aa1 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,6 @@ pom.xml.versionsBackup **/.env .instance_id rebuild-docker.sh -.run/ \ No newline at end of file +.run/ +.run/** +.run \ No newline at end of file diff --git a/application/pom.xml b/application/pom.xml index a5c0c43a8d..21fc355e4c 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -149,10 +149,6 @@ org.springframework.boot spring-boot-starter-websocket - - org.springframework.cloud - spring-cloud-starter-oauth2 - org.springframework.security spring-security-oauth2-client @@ -201,6 +197,14 @@ com.sun.mail javax.mail + + com.twilio.sdk + twilio + + + com.amazonaws + aws-java-sdk-sns + org.apache.curator curator-recipes diff --git a/application/src/main/data/json/demo/dashboards/gateways.json b/application/src/main/data/json/demo/dashboards/gateways.json index 34474ea890..0a18b10a52 100644 --- a/application/src/main/data/json/demo/dashboards/gateways.json +++ b/application/src/main/data/json/demo/dashboards/gateways.json @@ -955,7 +955,7 @@ }, "methodName": "gateway_restart", "methodParams": "{}", - "buttonText": "gateway restart" + "buttonText": "GATEWAY RESTART" }, "title": "New RPC Button", "dropShadow": true, diff --git a/application/src/main/data/json/demo/dashboards/raspberry_pi_gpio_demo_dashboard.json b/application/src/main/data/json/demo/dashboards/raspberry_pi_gpio_demo_dashboard.json deleted file mode 100644 index 1dfba7c8d4..0000000000 --- a/application/src/main/data/json/demo/dashboards/raspberry_pi_gpio_demo_dashboard.json +++ /dev/null @@ -1,700 +0,0 @@ -{ - "title": "Raspberry PI GPIO Demo Dashboard", - "configuration": { - "description": "Demo dashboard for Raspberry PI GPIO Demo", - "widgets": { - "602177f6-267b-cb87-4e8f-e23d7fb2f61c": { - "isSystemType": true, - "bundleAlias": "gpio_widgets", - "typeAlias": "raspberry_pi_gpio_control", - "type": "rpc", - "title": "New widget", - "sizeX": 6, - "sizeY": 10, - "config": { - "targetDeviceAliases": [], - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "0px", - "settings": { - "parseGpioStatusFunction": "return body[pin] === true;", - "gpioStatusChangeRequest": { - "method": "setGpioStatus", - "paramsBody": "{\n \"pin\": \"{$pin}\",\n \"enabled\": \"{$enabled}\"\n}" - }, - "requestTimeout": 500, - "switchPanelBackgroundColor": "#008a00", - "gpioStatusRequest": { - "method": "getGpioStatus", - "paramsBody": "{}" - }, - "gpioList": [ - { - "pin": 7, - "label": "GPIO 4 (GPCLK0)", - "row": 3, - "col": 0, - "_uniqueKey": 0 - }, - { - "pin": 11, - "label": "GPIO 17", - "row": 5, - "col": 0, - "_uniqueKey": 1 - }, - { - "pin": 12, - "label": "GPIO 18", - "row": 5, - "col": 1, - "_uniqueKey": 2 - }, - { - "_uniqueKey": 3, - "pin": 13, - "label": "GPIO 27", - "row": 6, - "col": 0 - }, - { - "_uniqueKey": 4, - "pin": 15, - "label": "GPIO 22", - "row": 7, - "col": 0 - }, - { - "_uniqueKey": 5, - "pin": 16, - "label": "GPIO 23", - "row": 7, - "col": 1 - }, - { - "_uniqueKey": 6, - "pin": 18, - "label": "GPIO 24", - "row": 8, - "col": 1 - }, - { - "_uniqueKey": 7, - "pin": 22, - "label": "GPIO 25", - "row": 10, - "col": 1 - }, - { - "_uniqueKey": 8, - "pin": 29, - "label": "GPIO 5", - "row": 14, - "col": 0 - }, - { - "_uniqueKey": 9, - "pin": 31, - "label": "GPIO 6", - "row": 15, - "col": 0 - }, - { - "_uniqueKey": 10, - "pin": 32, - "label": "GPIO 12", - "row": 15, - "col": 1 - }, - { - "_uniqueKey": 11, - "pin": 33, - "label": "GPIO 13", - "row": 16, - "col": 0 - }, - { - "_uniqueKey": 12, - "pin": 35, - "label": "GPIO 19", - "row": 17, - "col": 0 - }, - { - "_uniqueKey": 13, - "pin": 36, - "label": "GPIO 16", - "row": 17, - "col": 1 - }, - { - "_uniqueKey": 14, - "pin": 37, - "label": "GPIO 26", - "row": 18, - "col": 0 - }, - { - "_uniqueKey": 15, - "pin": 38, - "label": "GPIO 20", - "row": 18, - "col": 1 - }, - { - "_uniqueKey": 16, - "pin": 40, - "label": "GPIO 21", - "row": 19, - "col": 1 - } - ] - }, - "title": "Raspberry Pi GPIO Control Panel", - "datasources": [], - "targetDeviceAliasIds": [ - "f26b12b6-6938-e1a0-85ec-d88a1f23e382" - ] - }, - "row": 0, - "col": 0, - "id": "602177f6-267b-cb87-4e8f-e23d7fb2f61c" - }, - "3cca52a5-e874-eb43-b444-8efa01e663c8": { - "isSystemType": true, - "bundleAlias": "gpio_widgets", - "typeAlias": "raspberry_pi_gpio_panel", - "type": "latest", - "title": "New widget", - "sizeX": 7, - "sizeY": 10, - "config": { - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "0px", - "settings": { - "gpioList": [ - { - "pin": 1, - "label": "3.3V", - "row": 0, - "col": 0, - "color": "#fc9700", - "_uniqueKey": 0 - }, - { - "pin": 2, - "label": "5V", - "row": 0, - "col": 1, - "color": "#fb0000", - "_uniqueKey": 1 - }, - { - "pin": 3, - "label": "GPIO 2 (I2C1_SDA)", - "row": 1, - "col": 0, - "color": "#02fefb", - "_uniqueKey": 2 - }, - { - "color": "#fb0000", - "pin": 4, - "label": "5V", - "row": 1, - "col": 1 - }, - { - "color": "#02fefb", - "pin": 5, - "label": "GPIO 3 (I2C1_SCL)", - "row": 2, - "col": 0 - }, - { - "color": "#000000", - "pin": 6, - "label": "GND", - "row": 2, - "col": 1 - }, - { - "color": "#00fd00", - "pin": 7, - "label": "GPIO 4 (GPCLK0)", - "row": 3, - "col": 0 - }, - { - "color": "#fdfb00", - "pin": 8, - "label": "GPIO 14 (UART_TXD)", - "row": 3, - "col": 1 - }, - { - "color": "#000000", - "pin": 9, - "label": "GND", - "row": 4, - "col": 0 - }, - { - "color": "#fdfb00", - "pin": 10, - "label": "GPIO 15 (UART_RXD)", - "row": 4, - "col": 1 - }, - { - "color": "#00fd00", - "pin": 11, - "label": "GPIO 17", - "row": 5, - "col": 0 - }, - { - "color": "#00fd00", - "pin": 12, - "label": "GPIO 18", - "row": 5, - "col": 1 - }, - { - "color": "#00fd00", - "pin": 13, - "label": "GPIO 27", - "row": 6, - "col": 0 - }, - { - "color": "#000000", - "pin": 14, - "label": "GND", - "row": 6, - "col": 1 - }, - { - "color": "#00fd00", - "pin": 15, - "label": "GPIO 22", - "row": 7, - "col": 0 - }, - { - "color": "#00fd00", - "pin": 16, - "label": "GPIO 23", - "row": 7, - "col": 1 - }, - { - "color": "#fc9700", - "pin": 17, - "label": "3.3V", - "row": 8, - "col": 0 - }, - { - "color": "#00fd00", - "pin": 18, - "label": "GPIO 24", - "row": 8, - "col": 1 - }, - { - "color": "#fd01fd", - "pin": 19, - "label": "GPIO 10 (SPI_MOSI)", - "row": 9, - "col": 0 - }, - { - "color": "#000000", - "pin": 20, - "label": "GND", - "row": 9, - "col": 1 - }, - { - "color": "#fd01fd", - "pin": 21, - "label": "GPIO 9 (SPI_MISO)", - "row": 10, - "col": 0 - }, - { - "color": "#00fd00", - "pin": 22, - "label": "GPIO 25", - "row": 10, - "col": 1 - }, - { - "color": "#fd01fd", - "pin": 23, - "label": "GPIO 11 (SPI_SCLK)", - "row": 11, - "col": 0 - }, - { - "color": "#fd01fd", - "pin": 24, - "label": "GPIO 8 (SPI_CE0)", - "row": 11, - "col": 1 - }, - { - "color": "#000000", - "pin": 25, - "label": "GND", - "row": 12, - "col": 0 - }, - { - "color": "#fd01fd", - "pin": 26, - "label": "GPIO 7 (SPI_CE1)", - "row": 12, - "col": 1 - }, - { - "color": "#ffffff", - "pin": 27, - "label": "ID_SD", - "row": 13, - "col": 0 - }, - { - "color": "#ffffff", - "pin": 28, - "label": "ID_SC", - "row": 13, - "col": 1 - }, - { - "color": "#00fd00", - "pin": 29, - "label": "GPIO 5", - "row": 14, - "col": 0 - }, - { - "color": "#000000", - "pin": 30, - "label": "GND", - "row": 14, - "col": 1 - }, - { - "color": "#00fd00", - "pin": 31, - "label": "GPIO 6", - "row": 15, - "col": 0 - }, - { - "color": "#00fd00", - "pin": 32, - "label": "GPIO 12", - "row": 15, - "col": 1 - }, - { - "color": "#00fd00", - "pin": 33, - "label": "GPIO 13", - "row": 16, - "col": 0 - }, - { - "color": "#000000", - "pin": 34, - "label": "GND", - "row": 16, - "col": 1 - }, - { - "color": "#00fd00", - "pin": 35, - "label": "GPIO 19", - "row": 17, - "col": 0 - }, - { - "color": "#00fd00", - "pin": 36, - "label": "GPIO 16", - "row": 17, - "col": 1 - }, - { - "color": "#00fd00", - "pin": 37, - "label": "GPIO 26", - "row": 18, - "col": 0 - }, - { - "color": "#00fd00", - "pin": 38, - "label": "GPIO 20", - "row": 18, - "col": 1 - }, - { - "color": "#000000", - "pin": 39, - "label": "GND", - "row": 19, - "col": 0 - }, - { - "color": "#00fd00", - "pin": 40, - "label": "GPIO 21", - "row": 19, - "col": 1 - } - ], - "ledPanelBackgroundColor": "#008a00" - }, - "title": "Raspberry Pi GPIO Status Panel", - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "7", - "type": "attribute", - "label": "7", - "color": "#2196f3", - "settings": {}, - "_hash": 0.20925966435886978 - }, - { - "name": "11", - "type": "attribute", - "label": "11", - "color": "#4caf50", - "settings": {}, - "_hash": 0.330267349594344 - }, - { - "name": "12", - "type": "attribute", - "label": "12", - "color": "#f44336", - "settings": {}, - "_hash": 0.5040578704481748 - }, - { - "name": "13", - "type": "attribute", - "label": "13", - "color": "#ffc107", - "settings": {}, - "_hash": 0.588956328191639 - }, - { - "name": "15", - "type": "attribute", - "label": "15", - "color": "#607d8b", - "settings": {}, - "_hash": 0.9229040530336119 - }, - { - "name": "16", - "type": "attribute", - "label": "16", - "color": "#9c27b0", - "settings": {}, - "_hash": 0.8692315253041654 - }, - { - "name": "18", - "type": "attribute", - "label": "18", - "color": "#8bc34a", - "settings": {}, - "_hash": 0.41465562857521543 - }, - { - "name": "22", - "type": "attribute", - "label": "22", - "color": "#3f51b5", - "settings": {}, - "_hash": 0.36135260043112827 - }, - { - "name": "29", - "type": "attribute", - "label": "29", - "color": "#e91e63", - "settings": {}, - "_hash": 0.9904592276182183 - }, - { - "name": "31", - "type": "attribute", - "label": "31", - "color": "#ffeb3b", - "settings": {}, - "_hash": 0.038330985429919195 - }, - { - "name": "32", - "type": "attribute", - "label": "32", - "color": "#03a9f4", - "settings": {}, - "_hash": 0.4334683890135089 - }, - { - "name": "33", - "type": "attribute", - "label": "33", - "color": "#ff9800", - "settings": {}, - "_hash": 0.6487255992492305 - }, - { - "name": "35", - "type": "attribute", - "label": "35", - "color": "#673ab7", - "settings": {}, - "_hash": 0.971555321150732 - }, - { - "name": "36", - "type": "attribute", - "label": "36", - "color": "#cddc39", - "settings": {}, - "_hash": 0.7826129728424382 - }, - { - "name": "37", - "type": "attribute", - "label": "37", - "color": "#009688", - "settings": {}, - "_hash": 0.44925676517537627 - }, - { - "name": "38", - "type": "attribute", - "label": "38", - "color": "#795548", - "settings": {}, - "_hash": 0.051518155759787465 - }, - { - "name": "40", - "type": "attribute", - "label": "40", - "color": "#00bcd4", - "settings": {}, - "_hash": 0.8733296686871144 - } - ], - "name": "RPi", - "entityAliasId": "f26b12b6-6938-e1a0-85ec-d88a1f23e382" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - } - }, - "row": 0, - "col": 6, - "id": "3cca52a5-e874-eb43-b444-8efa01e663c8" - } - }, - "states": { - "default": { - "name": "Default", - "root": true, - "layouts": { - "main": { - "widgets": { - "602177f6-267b-cb87-4e8f-e23d7fb2f61c": { - "sizeX": 6, - "sizeY": 10, - "row": 0, - "col": 0 - }, - "3cca52a5-e874-eb43-b444-8efa01e663c8": { - "sizeX": 7, - "sizeY": 10, - "row": 0, - "col": 6 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "margins": [ - 10, - 10 - ], - "backgroundSizeMode": "100%" - } - } - } - } - }, - "entityAliases": { - "f26b12b6-6938-e1a0-85ec-d88a1f23e382": { - "id": "f26b12b6-6938-e1a0-85ec-d88a1f23e382", - "alias": "RPi", - "filter": { - "type": "entityName", - "resolveMultiple": false, - "entityType": "DEVICE", - "entityNameFilter": "Raspberry Pi Demo Device" - } - } - }, - "timewindow": { - "displayValue": "", - "selectedTab": 0, - "realtime": { - "interval": 1000, - "timewindowMs": 60000 - }, - "history": { - "historyType": 0, - "interval": 1000, - "timewindowMs": 60000, - "fixedTimewindow": { - "startTimeMs": 1498653734150, - "endTimeMs": 1498740134150 - } - }, - "aggregation": { - "type": "AVG", - "limit": 200 - } - }, - "settings": { - "stateControllerId": "default", - "showTitle": true, - "showDashboardsSelect": true, - "showEntitiesSelect": true, - "showDashboardTimewindow": true, - "showDashboardExport": true, - "toolbarAlwaysOpen": false - } - }, - "name": "Raspberry PI GPIO Demo Dashboard" -} \ No newline at end of file diff --git a/application/src/main/data/json/demo/dashboards/temperature___humidity_demo_dashboard.json b/application/src/main/data/json/demo/dashboards/temperature___humidity_demo_dashboard.json deleted file mode 100644 index 4d4f901558..0000000000 --- a/application/src/main/data/json/demo/dashboards/temperature___humidity_demo_dashboard.json +++ /dev/null @@ -1,264 +0,0 @@ -{ - "title": "Temperature & Humidity Demo Dashboard", - "configuration": { - "description": "Demo dashboard for sample applications that upload temperature and humidity received from DHT11 or DHT22 sensors", - "widgets": { - "03e06986-1c50-e9e4-267c-2bae930ad9a2": { - "isSystemType": true, - "bundleAlias": "digital_gauges", - "typeAlias": "digital_thermometer", - "type": "latest", - "title": "New widget", - "sizeX": 5, - "sizeY": 5, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "temperature", - "type": "timeseries", - "label": "temperature", - "color": "#2196f3", - "settings": {}, - "_hash": 0.3720839051412099 - } - ], - "name": "DHT11", - "entityAliasId": "63a93238-c13f-4403-4bcc-9ccc86bd6a62" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": false, - "backgroundColor": "#000000", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "0px", - "settings": { - "maxValue": 50, - "donutStartAngle": 90, - "showValue": true, - "showMinMax": true, - "gaugeWidthScale": 1, - "levelColors": [ - "#304ffe", - "#7e57c2", - "#ff4081", - "#d32f2f" - ], - "refreshAnimationType": "<>", - "refreshAnimationTime": 700, - "startAnimationType": "<>", - "startAnimationTime": 700, - "titleFont": { - "family": "RobotoDraft", - "size": 12, - "style": "normal", - "weight": "500" - }, - "labelFont": { - "family": "RobotoDraft", - "size": 8, - "style": "normal", - "weight": "500" - }, - "valueFont": { - "family": "Segment7Standard", - "style": "normal", - "weight": "500", - "size": 18 - }, - "minMaxFont": { - "family": "Segment7Standard", - "size": 12, - "style": "normal", - "weight": "500" - }, - "dashThickness": 1.5, - "decimals": 0, - "minValue": 0, - "units": "°C", - "gaugeColor": "#333333", - "neonGlowBrightness": 35, - "gaugeType": "donut", - "showTitle": false - }, - "title": "Temperature" - }, - "row": 0, - "col": 0, - "id": "03e06986-1c50-e9e4-267c-2bae930ad9a2" - }, - "88808eb1-d381-9970-c852-e3499df68bd8": { - "isSystemType": true, - "bundleAlias": "digital_gauges", - "typeAlias": "digital_vertical_bar", - "type": "latest", - "title": "New widget", - "sizeX": 3, - "sizeY": 5, - "config": { - "datasources": [ - { - "type": "entity", - "dataKeys": [ - { - "name": "humidity", - "type": "timeseries", - "label": "humidity", - "color": "#2196f3", - "settings": {}, - "_hash": 0.9492802776509441 - } - ], - "name": "DHT11", - "entityAliasId": "63a93238-c13f-4403-4bcc-9ccc86bd6a62" - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": false, - "backgroundColor": "#000000", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "0px", - "settings": { - "maxValue": 100, - "donutStartAngle": 90, - "showValue": true, - "showMinMax": true, - "gaugeWidthScale": 0.75, - "levelColors": [ - "#3d5afe", - "#f44336" - ], - "refreshAnimationType": "<>", - "refreshAnimationTime": 700, - "startAnimationType": "<>", - "startAnimationTime": 700, - "titleFont": { - "family": "RobotoDraft", - "size": 12, - "style": "normal", - "weight": "500" - }, - "labelFont": { - "family": "RobotoDraft", - "size": 8, - "style": "normal", - "weight": "500" - }, - "valueFont": { - "family": "Segment7Standard", - "style": "normal", - "weight": "500", - "size": 14 - }, - "minMaxFont": { - "family": "Segment7Standard", - "size": 8, - "style": "normal", - "weight": "normal", - "color": "#cccccc" - }, - "neonGlowBrightness": 20, - "decimals": 0, - "showUnitTitle": true, - "gaugeColor": "#171a1c", - "gaugeType": "verticalBar", - "showTitle": false, - "minValue": 0, - "dashThickness": 1.2 - }, - "title": "Humidity" - }, - "row": 0, - "col": 5, - "id": "88808eb1-d381-9970-c852-e3499df68bd8" - } - }, - "states": { - "default": { - "name": "Default", - "root": true, - "layouts": { - "main": { - "widgets": { - "03e06986-1c50-e9e4-267c-2bae930ad9a2": { - "sizeX": 5, - "sizeY": 5, - "row": 0, - "col": 0 - }, - "88808eb1-d381-9970-c852-e3499df68bd8": { - "sizeX": 3, - "sizeY": 5, - "row": 0, - "col": 5 - } - }, - "gridSettings": { - "backgroundColor": "#eeeeee", - "color": "rgba(0,0,0,0.870588)", - "columns": 24, - "margins": [ - 10, - 10 - ], - "backgroundSizeMode": "100%" - } - } - } - } - }, - "entityAliases": { - "63a93238-c13f-4403-4bcc-9ccc86bd6a62": { - "id": "63a93238-c13f-4403-4bcc-9ccc86bd6a62", - "alias": "DHT11", - "filter": { - "type": "entityName", - "resolveMultiple": false, - "entityType": "DEVICE", - "entityNameFilter": "DHT11 Demo Device" - } - } - }, - "timewindow": { - "displayValue": "", - "selectedTab": 0, - "realtime": { - "interval": 1000, - "timewindowMs": 60000 - }, - "history": { - "historyType": 0, - "interval": 1000, - "timewindowMs": 60000, - "fixedTimewindow": { - "startTimeMs": 1498653790019, - "endTimeMs": 1498740190019 - } - }, - "aggregation": { - "type": "AVG", - "limit": 200 - } - }, - "settings": { - "stateControllerId": "default", - "showTitle": true, - "showDashboardsSelect": true, - "showEntitiesSelect": true, - "showDashboardTimewindow": true, - "showDashboardExport": true, - "toolbarAlwaysOpen": false - } - }, - "name": "Temperature & Humidity Demo Dashboard" -} \ No newline at end of file diff --git a/application/src/main/data/json/demo/dashboards/thermostats.json b/application/src/main/data/json/demo/dashboards/thermostats.json index 06edbcbef0..4f486effdf 100644 --- a/application/src/main/data/json/demo/dashboards/thermostats.json +++ b/application/src/main/data/json/demo/dashboards/thermostats.json @@ -147,9 +147,9 @@ "name": "Add", "icon": "add", "type": "customPretty", - "customHtml": "\n \n Add thermostat\n \n \n close\n \n \n \n \n \n \n \n Thermostat name\n \n \n Thermostat name is required.\n \n \n \n \n High temperature alarm\n \n \n High temperature threshold, °C\n \n \n High temperature threshold is required.\n \n \n \n \n Low humidity alarm\n \n \n \n Low humidity threshold, %\n \n \n Low humidity threshold is required.\n \n \n \n \n \n \n Create\n \n \n Cancel\n \n \n", + "customHtml": "\n \n Add thermostat\n \n \n close\n \n \n \n \n \n \n \n Thermostat name\n \n \n Thermostat name is required.\n \n \n \n \n High temperature alarm\n \n \n High temperature threshold, °C\n \n \n High temperature threshold is required.\n \n \n \n \n Low humidity alarm\n \n \n \n Low humidity threshold, %\n \n \n Low humidity threshold is required.\n \n \n \n \n \n \n Create\n \n \n Cancel\n \n \n", "customCss": ".add-entity-form{\n width: 300px;\n}\n", - "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet customDialog = $injector.get(widgetContext.servicesMap.get('customDialog'));\nlet deviceService = $injector.get(widgetContext.servicesMap.get('deviceService'));\nlet attributeService = $injector.get(widgetContext.servicesMap.get('attributeService'));\n\nopenAddEntityDialog();\n\nfunction openAddEntityDialog() {\n customDialog.customDialog(htmlTemplate, AddEntityDialogController).subscribe();\n}\n\nfunction AddEntityDialogController(instance) {\n let vm = instance;\n \n vm.addEntityFormGroup = vm.fb.group({\n entityName: ['', [vm.validators.required]],\n attributes: vm.fb.group({\n alarmTemperature: [false],\n thresholdTemperature: [{value: null, disabled: true}],\n alarmHumidity: [false],\n thresholdHumidity: [{value: null, disabled: true}]\n })\n });\n \n vm.addEntityFormGroup.get('attributes').get('alarmTemperature').valueChanges\n .subscribe(activate => {\n if (activate) {\n vm.addEntityFormGroup.get('attributes').get('thresholdTemperature').enable();\n } else {\n vm.addEntityFormGroup.get('attributes').get('thresholdTemperature').disable();\n }\n });\n \n vm.addEntityFormGroup.get('attributes').get('alarmHumidity').valueChanges\n .subscribe(activate => {\n if (activate) {\n vm.addEntityFormGroup.get('attributes').get('thresholdHumidity').enable();\n } else {\n vm.addEntityFormGroup.get('attributes').get('thresholdHumidity').disable();\n }\n });\n\n vm.save = function() {\n vm.addEntityFormGroup.markAsPristine();\n saveEntityObservable().subscribe(\n function (entity) {\n saveAttributes(entity.id).subscribe(\n function () {\n widgetContext.updateAliases();\n vm.dialogRef.close(null);\n }\n );\n }\n );\n };\n \n vm.cancel = function() {\n vm.dialogRef.close(null);\n };\n \n function saveEntityObservable() {\n const formValues = vm.addEntityFormGroup.value;\n let entity = {\n name: formValues.entityName,\n type: \"thermostat\"\n };\n return deviceService.saveDevice(entity);\n }\n \n function saveAttributes(entityId) {\n let attributes = vm.addEntityFormGroup.get('attributes').value;\n let attributesArray = [];\n for (let key in attributes) {\n if(attributes[key] !== null) {\n attributesArray.push({key: key, value: attributes[key]});\n }\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId, \"SERVER_SCOPE\", attributesArray);\n } else {\n return widgetContext.rxjs.of([]);\n }\n }\n}", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet customDialog = $injector.get(widgetContext.servicesMap.get('customDialog'));\nlet deviceService = $injector.get(widgetContext.servicesMap.get('deviceService'));\nlet attributeService = $injector.get(widgetContext.servicesMap.get('attributeService'));\n\nopenAddEntityDialog();\n\nfunction openAddEntityDialog() {\n customDialog.customDialog(htmlTemplate, AddEntityDialogController).subscribe();\n}\n\nfunction AddEntityDialogController(instance) {\n let vm = instance;\n \n vm.addEntityFormGroup = vm.fb.group({\n entityName: ['', [vm.validators.required]],\n attributes: vm.fb.group({\n temperatureAlarmFlag: [false],\n temperatureAlarmThreshold: [{value: null, disabled: true}],\n humidityAlarmFlag: [false],\n humidityAlarmThreshold: [{value: null, disabled: true}]\n })\n });\n \n vm.addEntityFormGroup.get('attributes').get('temperatureAlarmFlag').valueChanges\n .subscribe(activate => {\n if (activate) {\n vm.addEntityFormGroup.get('attributes').get('temperatureAlarmThreshold').enable();\n } else {\n vm.addEntityFormGroup.get('attributes').get('temperatureAlarmThreshold').disable();\n }\n });\n \n vm.addEntityFormGroup.get('attributes').get('humidityAlarmFlag').valueChanges\n .subscribe(activate => {\n if (activate) {\n vm.addEntityFormGroup.get('attributes').get('humidityAlarmThreshold').enable();\n } else {\n vm.addEntityFormGroup.get('attributes').get('humidityAlarmThreshold').disable();\n }\n });\n\n vm.save = function() {\n vm.addEntityFormGroup.markAsPristine();\n saveEntityObservable().subscribe(\n function (entity) {\n saveAttributes(entity.id).subscribe(\n function () {\n widgetContext.updateAliases();\n vm.dialogRef.close(null);\n }\n );\n }\n );\n };\n \n vm.cancel = function() {\n vm.dialogRef.close(null);\n };\n \n function saveEntityObservable() {\n const formValues = vm.addEntityFormGroup.value;\n let entity = {\n name: formValues.entityName,\n type: \"thermostat\"\n };\n return deviceService.saveDevice(entity);\n }\n \n function saveAttributes(entityId) {\n let attributes = vm.addEntityFormGroup.get('attributes').value;\n let attributesArray = [];\n for (let key in attributes) {\n if(attributes[key] !== null) {\n attributesArray.push({key: key, value: attributes[key]});\n }\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId, \"SERVER_SCOPE\", attributesArray);\n } else {\n return widgetContext.rxjs.of([]);\n }\n }\n}", "customResources": [], "id": "8ab5a518-67d2-b6a2-956d-81fd512294b2" } @@ -167,9 +167,9 @@ "name": "Edit", "icon": "edit", "type": "customPretty", - "customHtml": "\n \n Edit thermostat {{entityName}}\n \n \n close\n \n \n \n \n \n \n \n Thermostat name\n \n \n \n \n High temperature alarm\n \n \n High temperature threshold, °C\n \n \n High temperature threshold is required.\n \n \n\n \n Low humidity alarm\n \n\n \n Low humidity threshold, %\n \n \n Low humidity threshold is required.\n \n \n \n \n \n \n Save\n \n \n Cancel\n \n \n", + "customHtml": "\n \n Edit thermostat {{entityName}}\n \n \n close\n \n \n \n \n \n \n \n Thermostat name\n \n \n \n \n High temperature alarm\n \n \n High temperature threshold, °C\n \n \n High temperature threshold is required.\n \n \n\n \n Low humidity alarm\n \n\n \n Low humidity threshold, %\n \n \n Low humidity threshold is required.\n \n \n \n \n \n \n Save\n \n \n Cancel\n \n \n", "customCss": ".edit-entity-form{\n width: 300px;\n}", - "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet customDialog = $injector.get(widgetContext.servicesMap.get('customDialog'));\nlet attributeService = $injector.get(widgetContext.servicesMap.get('attributeService'));\n\nopenEditEntityDialog();\n\nfunction openEditEntityDialog() {\n customDialog.customDialog(htmlTemplate, EditEntityDialogController).subscribe();\n}\n\nfunction EditEntityDialogController(instance) {\n let vm = instance;\n \n vm.entityId = entityId;\n vm.entityName = entityName;\n vm.attributes = {};\n \n vm.editEntityFormGroup = vm.fb.group({\n entityName: [''],\n attributes: vm.fb.group({\n alarmTemperature: [false],\n thresholdTemperature: [{value: null, disabled: true}],\n alarmHumidity: [false],\n thresholdHumidity: [{value: null, disabled: true}]\n })\n });\n \n vm.editEntityFormGroup.get('attributes').get('alarmTemperature').valueChanges\n .subscribe(activate => {\n if (activate) {\n vm.editEntityFormGroup.get('attributes').get('thresholdTemperature').enable();\n } else {\n vm.editEntityFormGroup.get('attributes').get('thresholdTemperature').disable();\n }\n });\n \n vm.editEntityFormGroup.get('attributes').get('alarmHumidity').valueChanges\n .subscribe(activate => {\n if (activate) {\n vm.editEntityFormGroup.get('attributes').get('thresholdHumidity').enable();\n } else {\n vm.editEntityFormGroup.get('attributes').get('thresholdHumidity').disable();\n }\n });\n \n \n getEntityInfo();\n \n \n vm.save = function() {\n vm.editEntityFormGroup.markAsPristine();\n saveAttributes(entityId).subscribe(\n function () {\n vm.dialogRef.close(null);\n }\n );\n };\n \n vm.cancel = function() {\n vm.dialogRef.close(null);\n };\n \n function getEntityAttributes(attributes) {\n for (var i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value;\n }\n }\n \n function getEntityInfo() {\n attributeService.getEntityAttributes(entityId, 'SERVER_SCOPE').subscribe(\n function (attributes) {\n getEntityAttributes(attributes);\n vm.editEntityFormGroup.patchValue({\n entityName: vm.entityName,\n attributes: vm.attributes\n });\n // if(vm.attributes.alarmTemperature) {\n // vm.editEntityFormGroup.get('attributes').get('thresholdTemperature').enable();\n // }\n // if(vm.attributes.alarmHumidity) {\n // vm.editEntityFormGroup.get('attributes').get('thresholdHumidity').enable();\n // }\n }\n );\n }\n \n function saveAttributes(entityId) {\n let attributes = vm.editEntityFormGroup.get('attributes').value;\n let attributesArray = [];\n for (let key in attributes) {\n if (attributes[key] !== vm.attributes[key]) {\n attributesArray.push({key: key, value: attributes[key]});\n }\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId, \"SERVER_SCOPE\", attributesArray);\n } else {\n return widgetContext.rxjs.of([]);\n }\n }\n}", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet customDialog = $injector.get(widgetContext.servicesMap.get('customDialog'));\nlet attributeService = $injector.get(widgetContext.servicesMap.get('attributeService'));\n\nopenEditEntityDialog();\n\nfunction openEditEntityDialog() {\n customDialog.customDialog(htmlTemplate, EditEntityDialogController).subscribe();\n}\n\nfunction EditEntityDialogController(instance) {\n let vm = instance;\n \n vm.entityId = entityId;\n vm.entityName = entityName;\n vm.attributes = {};\n \n vm.editEntityFormGroup = vm.fb.group({\n entityName: [''],\n attributes: vm.fb.group({\n temperatureAlarmFlag: [false],\n temperatureAlarmThreshold: [{value: null, disabled: true}],\n humidityAlarmFlag: [false],\n humidityAlarmThreshold: [{value: null, disabled: true}]\n })\n });\n \n vm.editEntityFormGroup.get('attributes').get('temperatureAlarmFlag').valueChanges\n .subscribe(activate => {\n if (activate) {\n vm.editEntityFormGroup.get('attributes').get('temperatureAlarmThreshold').enable();\n } else {\n vm.editEntityFormGroup.get('attributes').get('temperatureAlarmThreshold').disable();\n }\n });\n \n vm.editEntityFormGroup.get('attributes').get('humidityAlarmFlag').valueChanges\n .subscribe(activate => {\n if (activate) {\n vm.editEntityFormGroup.get('attributes').get('humidityAlarmThreshold').enable();\n } else {\n vm.editEntityFormGroup.get('attributes').get('humidityAlarmThreshold').disable();\n }\n });\n \n \n getEntityInfo();\n \n \n vm.save = function() {\n vm.editEntityFormGroup.markAsPristine();\n saveAttributes(entityId).subscribe(\n function () {\n vm.dialogRef.close(null);\n }\n );\n };\n \n vm.cancel = function() {\n vm.dialogRef.close(null);\n };\n \n function getEntityAttributes(attributes) {\n for (var i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value;\n }\n }\n \n function getEntityInfo() {\n attributeService.getEntityAttributes(entityId, 'SERVER_SCOPE').subscribe(\n function (attributes) {\n getEntityAttributes(attributes);\n vm.editEntityFormGroup.patchValue({\n entityName: vm.entityName,\n attributes: vm.attributes\n });\n // if(vm.attributes.temperatureAlarmFlag) {\n // vm.editEntityFormGroup.get('attributes').get('temperatureAlarmThreshold').enable();\n // }\n // if(vm.attributes.humidityAlarmFlag) {\n // vm.editEntityFormGroup.get('attributes').get('humidityAlarmThreshold').enable();\n // }\n }\n );\n }\n \n function saveAttributes(entityId) {\n let attributes = vm.editEntityFormGroup.get('attributes').value;\n let attributesArray = [];\n for (let key in attributes) {\n if (attributes[key] !== vm.attributes[key]) {\n attributesArray.push({key: key, value: attributes[key]});\n }\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId, \"SERVER_SCOPE\", attributesArray);\n } else {\n return widgetContext.rxjs.of([]);\n }\n }\n}", "customResources": [], "id": "7506576f-87ba-d3a0-88fb-e304d451776d" }, @@ -219,8 +219,8 @@ "defaultPageSize": 10, "defaultSortOrder": "-createdTime", "enableSelectColumnDisplay": false, - "enableStatusFilter": true, - "alarmsTitle": "Alarms" + "alarmsTitle": "Alarms", + "enableFilter": true }, "title": "New Alarms table", "dropShadow": true, @@ -234,6 +234,9 @@ "showLegend": false, "alarmSource": { "type": "entity", + "name": "alarms", + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e", + "filterId": null, "dataKeys": [ { "name": "createdTime", @@ -275,9 +278,7 @@ "settings": {}, "_hash": 0.7977920750136249 } - ], - "entityAliasId": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", - "name": "alarms" + ] }, "alarmSearchStatus": "ANY", "alarmsPollingInterval": 5, @@ -549,7 +550,7 @@ "type": "entity", "dataKeys": [ { - "name": "alarmTemperature", + "name": "temperatureAlarmFlag", "type": "attribute", "label": "High temperature alarm", "color": "#4caf50", @@ -564,7 +565,7 @@ "_hash": 0.8725278440159361 }, { - "name": "thresholdTemperature", + "name": "temperatureAlarmThreshold", "type": "attribute", "label": "High temperature threshold, °C", "color": "#f44336", @@ -575,12 +576,12 @@ "isEditable": "editable", "dataKeyHidden": false, "step": 1, - "disabledOnDataKey": "alarmTemperature" + "disabledOnDataKey": "temperatureAlarmFlag" }, "_hash": 0.7316078472857874 }, { - "name": "alarmHumidity", + "name": "humidityAlarmFlag", "type": "attribute", "label": "Low humidity alarm", "color": "#ffc107", @@ -595,7 +596,7 @@ "_hash": 0.5339673667431057 }, { - "name": "thresholdHumidity", + "name": "humidityAlarmThreshold", "type": "attribute", "label": "Low humidity threshold, %", "color": "#607d8b", @@ -606,7 +607,7 @@ "isEditable": "editable", "dataKeyHidden": false, "step": 1, - "disabledOnDataKey": "alarmHumidity" + "disabledOnDataKey": "humidityAlarmFlag" }, "_hash": 0.2687091190358901 } @@ -1031,7 +1032,8 @@ "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;", "useLabelFunction": true, "provider": "openstreet-map", - "draggableMarker": true + "draggableMarker": true, + "editablePolygon": true }, "title": "New Markers Placement - OpenStreetMap", "dropShadow": true, @@ -1062,61 +1064,6 @@ "displayTimewindow": true }, "id": "0a430429-9078-9ae6-2b67-e4a15a2bf8bf" - }, - "f4bb2f2d-0164-60bc-f3e8-9b1e7b5a59b3": { - "isSystemType": true, - "bundleAlias": "input_widgets", - "typeAlias": "update_double_timeseries", - "type": "latest", - "title": "New widget", - "sizeX": 7.5, - "sizeY": 3, - "config": { - "datasources": [ - { - "type": "entity", - "name": null, - "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547", - "dataKeys": [ - { - "name": "temperature", - "type": "timeseries", - "label": "temperature", - "color": "#2196f3", - "settings": {}, - "_hash": 0.4164505192982848 - } - ] - } - ], - "timewindow": { - "realtime": { - "timewindowMs": 60000 - } - }, - "showTitle": true, - "backgroundColor": "#fff", - "color": "rgba(0, 0, 0, 0.87)", - "padding": "8px", - "settings": { - "showResultMessage": true, - "showLabel": true - }, - "title": "New Update double timeseries", - "dropShadow": true, - "enableFullscreen": false, - "widgetStyle": {}, - "titleStyle": { - "fontSize": "16px", - "fontWeight": 400 - }, - "useDashboardTimewindow": true, - "showLegend": false, - "actions": {} - }, - "row": 0, - "col": 0, - "id": "f4bb2f2d-0164-60bc-f3e8-9b1e7b5a59b3" } }, "states": { @@ -1215,12 +1162,6 @@ "sizeY": 6, "row": 6, "col": 0 - }, - "f4bb2f2d-0164-60bc-f3e8-9b1e7b5a59b3": { - "sizeX": 7.5, - "sizeY": 3, - "row": 12, - "col": 0 } }, "gridSettings": { @@ -1257,16 +1198,6 @@ "stateEntityParamName": null, "defaultStateEntity": null } - }, - "ce27a9d0-93bf-b7a4-054d-d0369a8cf813": { - "id": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", - "alias": "Thermostat-alarm", - "filter": { - "type": "entityName", - "resolveMultiple": false, - "entityType": "ASSET", - "entityNameFilter": "Thermostat Alarms" - } } }, "timewindow": { @@ -1301,7 +1232,8 @@ "showDashboardTimewindow": true, "showDashboardExport": true, "toolbarAlwaysOpen": true - } + }, + "filters": {} }, "name": "Thermostats" } \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json deleted file mode 100644 index 067685d775..0000000000 --- a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json +++ /dev/null @@ -1,190 +0,0 @@ -{ - "ruleChain": { - "additionalInfo": null, - "name": "Root Rule Chain", - "type": "CORE", - "firstRuleNodeId": null, - "root": true, - "debugMode": false, - "configuration": null - }, - "metadata": { - "firstNodeIndex": 3, - "nodes": [ - { - "additionalInfo": { - "layoutX": 1069, - "layoutY": 267 - }, - "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", - "name": "Is Thermostat?", - "debugMode": false, - "configuration": { - "jsScript": "return msg.id.entityType === \"DEVICE\" && msg.type === \"thermostat\";" - } - }, - { - "additionalInfo": { - "layoutX": 824, - "layoutY": 156 - }, - "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", - "name": "Save Timeseries", - "debugMode": false, - "configuration": { - "defaultTTL": 0 - } - }, - { - "additionalInfo": { - "layoutX": 825, - "layoutY": 52 - }, - "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", - "name": "Save Client Attributes", - "debugMode": false, - "configuration": { - "scope": "CLIENT_SCOPE", - "notifyDevice": "false" - } - }, - { - "additionalInfo": { - "layoutX": 347, - "layoutY": 149 - }, - "type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode", - "name": "Message Type Switch", - "debugMode": false, - "configuration": { - "version": 0 - } - }, - { - "additionalInfo": { - "layoutX": 839, - "layoutY": 345 - }, - "type": "org.thingsboard.rule.engine.action.TbLogNode", - "name": "Log RPC from Device", - "debugMode": false, - "configuration": { - "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" - } - }, - { - "additionalInfo": { - "layoutX": 832, - "layoutY": 407 - }, - "type": "org.thingsboard.rule.engine.action.TbLogNode", - "name": "Log Other", - "debugMode": false, - "configuration": { - "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" - } - }, - { - "additionalInfo": { - "layoutX": 825, - "layoutY": 468 - }, - "type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode", - "name": "RPC Call Request", - "debugMode": false, - "configuration": { - "timeoutInSeconds": 60 - } - }, - { - "additionalInfo": { - "layoutX": 1069, - "layoutY": 90 - }, - "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", - "name": "Is Thermostat?", - "debugMode": false, - "configuration": { - "jsScript": "return metadata[\"deviceType\"] === \"thermostat\";" - } - }, - { - "additionalInfo": { - "layoutX": 1090, - "layoutY": 360 - }, - "type": "org.thingsboard.rule.engine.action.TbCreateRelationNode", - "name": "Relate to Asset", - "debugMode": false, - "configuration": { - "direction": "FROM", - "relationType": "ToAlarmPropagationAsset", - "entityType": "ASSET", - "entityNamePattern": "Thermostat Alarms", - "entityTypePattern": "AlarmPropagationAsset", - "entityCacheExpiration": 300, - "createEntityIfNotExists": true, - "changeOriginatorToRelatedEntity": false, - "removeCurrentRelations": false - } - } - ], - "connections": [ - { - "fromIndex": 0, - "toIndex": 8, - "type": "True" - }, - { - "fromIndex": 1, - "toIndex": 7, - "type": "Success" - }, - { - "fromIndex": 3, - "toIndex": 5, - "type": "Other" - }, - { - "fromIndex": 3, - "toIndex": 2, - "type": "Post attributes" - }, - { - "fromIndex": 3, - "toIndex": 1, - "type": "Post telemetry" - }, - { - "fromIndex": 3, - "toIndex": 4, - "type": "RPC Request from Device" - }, - { - "fromIndex": 3, - "toIndex": 6, - "type": "RPC Request to Device" - }, - { - "fromIndex": 3, - "toIndex": 0, - "type": "Entity Created" - } - ], - "ruleChainConnections": [ - { - "fromIndex": 7, - "targetRuleChainId": { - "entityType": "RULE_CHAIN", - "id": "25e26570-89ed-11ea-a650-cd6e14e633bd" - }, - "additionalInfo": { - "layoutX": 1109, - "layoutY": 182, - "ruleChainNodeId": "rule-chain-node-10" - }, - "type": "True" - } - ] - } -} \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json deleted file mode 100644 index e3ba2e3192..0000000000 --- a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "ruleChain": { - "additionalInfo": null, - "name": "Thermostat Alarms", - "type": "CORE", - "firstRuleNodeId": null, - "root": false, - "debugMode": false, - "configuration": null - }, - "metadata": { - "firstNodeIndex": 5, - "nodes": [ - { - "additionalInfo": { - "layoutX": 929, - "layoutY": 67 - }, - "type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode", - "name": "Create Temp Alarm", - "debugMode": false, - "configuration": { - "alarmType": "High Temperature", - "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.temperature;\nreturn details;", - "severity": "MAJOR", - "propagate": true, - "useMessageAlarmData": false, - "relationTypes": [ - "ToAlarmPropagationAsset" - ] - } - }, - { - "additionalInfo": { - "layoutX": 930, - "layoutY": 201 - }, - "type": "org.thingsboard.rule.engine.action.TbClearAlarmNode", - "name": "Clear Temp Alarm", - "debugMode": false, - "configuration": { - "alarmType": "High Temperature", - "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;" - } - }, - { - "additionalInfo": { - "layoutX": 930, - "layoutY": 131 - }, - "type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode", - "name": "Create Humidity Alarm", - "debugMode": false, - "configuration": { - "alarmType": "Low Humidity", - "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.humidity;\nreturn details;", - "severity": "MINOR", - "propagate": true, - "useMessageAlarmData": false, - "relationTypes": [ - "ToAlarmPropagationAsset" - ] - } - }, - { - "additionalInfo": { - "layoutX": 929, - "layoutY": 275 - }, - "type": "org.thingsboard.rule.engine.action.TbClearAlarmNode", - "name": "Clear Humidity Alarm", - "debugMode": false, - "configuration": { - "alarmType": "Low Humidity", - "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;" - } - }, - { - "additionalInfo": { - "layoutX": 586, - "layoutY": 148 - }, - "type": "org.thingsboard.rule.engine.filter.TbJsSwitchNode", - "name": "Check Alarms", - "debugMode": false, - "configuration": { - "jsScript": "var relations = [];\nif(metadata[\"ss_alarmTemperature\"] === \"true\"){\n if(msg.temperature > metadata[\"ss_thresholdTemperature\"]){\n relations.push(\"NewTempAlarm\");\n } else {\n relations.push(\"ClearTempAlarm\");\n }\n}\nif(metadata[\"ss_alarmHumidity\"] === \"true\"){\n if(msg.humidity < metadata[\"ss_thresholdHumidity\"]){\n relations.push(\"NewHumidityAlarm\");\n } else {\n relations.push(\"ClearHumidityAlarm\");\n }\n}\n\nreturn relations;" - } - }, - { - "additionalInfo": { - "layoutX": 321, - "layoutY": 149 - }, - "type": "org.thingsboard.rule.engine.metadata.TbGetAttributesNode", - "name": "Fetch Configuration", - "debugMode": false, - "configuration": { - "clientAttributeNames": [], - "sharedAttributeNames": [], - "serverAttributeNames": [ - "alarmTemperature", - "thresholdTemperature", - "alarmHumidity", - "thresholdHumidity" - ], - "latestTsKeyNames": [], - "tellFailureIfAbsent": false, - "getLatestValueWithTs": false - } - } - ], - "connections": [ - { - "fromIndex": 4, - "toIndex": 0, - "type": "NewTempAlarm" - }, - { - "fromIndex": 4, - "toIndex": 1, - "type": "ClearTempAlarm" - }, - { - "fromIndex": 4, - "toIndex": 2, - "type": "NewHumidityAlarm" - }, - { - "fromIndex": 4, - "toIndex": 3, - "type": "ClearHumidityAlarm" - }, - { - "fromIndex": 5, - "toIndex": 4, - "type": "Success" - } - ], - "ruleChainConnections": null - } -} \ No newline at end of file diff --git a/application/src/main/data/json/system/oauth2_config_templates/facebook_config.json b/application/src/main/data/json/system/oauth2_config_templates/facebook_config.json new file mode 100644 index 0000000000..04847a9da6 --- /dev/null +++ b/application/src/main/data/json/system/oauth2_config_templates/facebook_config.json @@ -0,0 +1,23 @@ +{ + "providerId": "Facebook", + "accessTokenUri": "https://graph.facebook.com/v2.8/oauth/access_token", + "authorizationUri": "https://www.facebook.com/v2.8/dialog/oauth", + "scope": ["email","public_profile"], + "jwkSetUri": null, + "userInfoUri": "https://graph.facebook.com/me?fields=id,name,first_name,last_name,email", + "clientAuthenticationMethod": "BASIC", + "userNameAttributeName": "email", + "mapperConfig": { + "type": "BASIC", + "basic": { + "emailAttributeKey": "email", + "firstNameAttributeKey": "first_name", + "lastNameAttributeKey": "last_name", + "tenantNameStrategy": "DOMAIN" + } + }, + "comment": null, + "loginButtonIcon": "facebook-logo", + "loginButtonLabel": "Facebook", + "helpLink": "https://developers.facebook.com/docs/facebook-login/web#logindialog" +} diff --git a/application/src/main/data/json/system/oauth2_config_templates/github_config.json b/application/src/main/data/json/system/oauth2_config_templates/github_config.json new file mode 100644 index 0000000000..439043d961 --- /dev/null +++ b/application/src/main/data/json/system/oauth2_config_templates/github_config.json @@ -0,0 +1,21 @@ +{ + "providerId": "Github", + "accessTokenUri": "https://github.com/login/oauth/access_token", + "authorizationUri": "https://github.com/login/oauth/authorize", + "scope": ["read:user","user:email"], + "jwkSetUri": null, + "userInfoUri": "https://api.github.com/user", + "clientAuthenticationMethod": "BASIC", + "userNameAttributeName": "login", + "mapperConfig": { + "type": "GITHUB", + "basic": { + "firstNameAttributeKey": "name", + "tenantNameStrategy": "DOMAIN" + } + }, + "comment": "In order to log into ThingsBoard you need to have user's email. You may configure and use Custom OAuth2 Mapper to get email information. Please refer to Github Documentation", + "loginButtonIcon": "github-logo", + "loginButtonLabel": "Github", + "helpLink": "https://docs.github.com/en/developers/apps/creating-an-oauth-app" +} diff --git a/application/src/main/data/json/system/oauth2_config_templates/google_config.json b/application/src/main/data/json/system/oauth2_config_templates/google_config.json new file mode 100644 index 0000000000..f8626439ba --- /dev/null +++ b/application/src/main/data/json/system/oauth2_config_templates/google_config.json @@ -0,0 +1,24 @@ +{ + "providerId": "Google", + "additionalInfo": null, + "accessTokenUri": "https://oauth2.googleapis.com/token", + "authorizationUri": "https://accounts.google.com/o/oauth2/v2/auth", + "scope": ["email","openid","profile"], + "jwkSetUri": "https://www.googleapis.com/oauth2/v3/certs", + "userInfoUri": "https://openidconnect.googleapis.com/v1/userinfo", + "clientAuthenticationMethod": "BASIC", + "userNameAttributeName": "email", + "mapperConfig": { + "type": "BASIC", + "basic": { + "emailAttributeKey": "email", + "firstNameAttributeKey": "given_name", + "lastNameAttributeKey": "family_name", + "tenantNameStrategy": "DOMAIN" + } + }, + "comment": null, + "loginButtonIcon": "google-logo", + "loginButtonLabel": "Google", + "helpLink": "https://developers.google.com/adwords/api/docs/guides/authentication" +} diff --git a/application/src/main/data/json/system/widget_bundles/charts.json b/application/src/main/data/json/system/widget_bundles/charts.json index 22c9d20a23..1010013689 100644 --- a/application/src/main/data/json/system/widget_bundles/charts.json +++ b/application/src/main/data/json/system/widget_bundles/charts.json @@ -19,7 +19,7 @@ ], "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n var barData = {\n labels: [],\n datasets: []\n };\n \n for (var i = 0; i < self.ctx.datasources.length; i++) {\n var datasource = self.ctx.datasources[i];\n for (var d = 0; d < datasource.dataKeys.length; d++) {\n var dataset = {\n label: datasource.dataKeys[d].label,\n data: [0],\n backgroundColor: [datasource.dataKeys[d].color],\n borderColor: [datasource.dataKeys[d].color],\n borderWidth: 1\n }\n barData.datasets.push(dataset);\n }\n }\n\n var ctx = $('#barChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'bar',\n data: barData,\n options: {\n responsive: false,\n maintainAspectRatio: false,\n scales: {\n yAxes: [{\n ticks: {\n beginAtZero:true\n }\n }]\n }\n }\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n var c = 0;\n for (var i = 0; i < self.ctx.chart.data.datasets.length; i++) {\n var dataset = self.ctx.chart.data.datasets[i];\n var cellData = self.ctx.data[i]; \n if (cellData.data.length > 0) {\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = tvPair[1];\n dataset.data[0] = parseFloat(value);\n }\n }\n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n self.ctx.chart.resize();\n}\n\n", + "controllerScript": "self.onInit = function() {\n var barData = {\n labels: [],\n datasets: []\n };\n \n for (var i = 0; i < self.ctx.datasources.length; i++) {\n var datasource = self.ctx.datasources[i];\n for (var d = 0; d < datasource.dataKeys.length; d++) {\n var dataKey = datasource.dataKeys[d];\n var units = dataKey.units && dataKey.units.length ? dataKey.units : self.ctx.units;\n units = units ? (' (' + units + ')') : '';\n var dataset = {\n label: dataKey.label + units,\n data: [0],\n backgroundColor: [dataKey.color],\n borderColor: [dataKey.color],\n borderWidth: 1\n }\n barData.datasets.push(dataset);\n }\n }\n\n var ctx = $('#barChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'bar',\n data: barData,\n options: {\n responsive: false,\n maintainAspectRatio: false,\n scales: {\n yAxes: [{\n ticks: {\n beginAtZero:true\n }\n }]\n }\n }\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n var c = 0;\n for (var i = 0; i < self.ctx.chart.data.datasets.length; i++) {\n var dataset = self.ctx.chart.data.datasets[i];\n var cellData = self.ctx.data[i]; \n if (cellData.data.length > 0) {\n var decimals;\n if (typeof cellData.dataKey.decimals !== 'undefined' \n && cellData.dataKey.decimals !== null ) {\n decimals = cellData.dataKey.decimals; \n } else {\n decimals = self.ctx.decimals;\n }\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = self.ctx.utils.formatValue(tvPair[1], decimals);\n dataset.data[0] = parseFloat(value);\n }\n }\n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n self.ctx.chart.resize();\n}\n\n", "settingsSchema": "{}", "dataKeySettingsSchema": "{}\n", "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"First\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Second\",\"color\":\"#4caf50\",\"settings\":{},\"_hash\":0.545701115289893,\"funcBody\":\"var value = (prevValue-20) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+20;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Third\",\"color\":\"#f44336\",\"settings\":{},\"_hash\":0.2592906835158064,\"funcBody\":\"var value = (prevValue-40) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+40;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fourth\",\"color\":\"#ffc107\",\"settings\":{},\"_hash\":0.12880275585455747,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Bars - Chart.js\"}" @@ -55,7 +55,7 @@ ], "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n var pieData = {\n labels: [],\n datasets: []\n };\n\n var dataset = {\n data: [],\n backgroundColor: [],\n borderColor: [],\n borderWidth: [],\n hoverBackgroundColor: []\n }\n \n var borderColor = self.ctx.settings.borderColor || '#fff';\n var borderWidth = typeof self.ctx.settings.borderWidth !== 'undefined' ? self.ctx.settings.borderWidth : 5;\n \n pieData.datasets.push(dataset);\n \n for (var i=0; i < self.ctx.data.length; i++) {\n var dataKey = self.ctx.data[i].dataKey;\n pieData.labels.push(dataKey.label);\n dataset.data.push(0);\n var hoverBackgroundColor = tinycolor(dataKey.color).lighten(15);\n dataset.backgroundColor.push(dataKey.color);\n dataset.borderColor.push(borderColor);\n dataset.borderWidth.push(borderWidth);\n dataset.hoverBackgroundColor.push(hoverBackgroundColor.toRgbString());\n }\n\n var options = {\n responsive: false,\n maintainAspectRatio: false,\n legend: {\n display: true,\n labels: {\n fontColor: '#666'\n }\n },\n tooltips: {\n callbacks: {\n label: function(tooltipItem, data) {\n var label = data.labels[tooltipItem.index];\n var value = data.datasets[tooltipItem.datasetIndex].data[tooltipItem.index];\n var content = label + ': ' + value;\n var units = self.ctx.settings.units ? self.ctx.settings.units : self.ctx.units;\n if (units) {\n content += ' ' + units;\n } \n return content;\n }\n }\n }\n };\n\n if (self.ctx.settings.legend) {\n options.legend.display = self.ctx.settings.legend.display !== false;\n options.legend.labels.fontColor = self.ctx.settings.legend.labelsFontColor || '#666';\n }\n\n var ctx = $('#pieChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'doughnut',\n data: pieData,\n options: options\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.data.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData.data.length > 0) {\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = tvPair[1];\n self.ctx.chart.data.datasets[0].data[i] = parseFloat(value);\n }\n }\n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n self.ctx.chart.resize();\n}\n\n", + "controllerScript": "self.onInit = function() {\n var pieData = {\n labels: [],\n datasets: []\n };\n\n var dataset = {\n data: [],\n backgroundColor: [],\n borderColor: [],\n borderWidth: [],\n hoverBackgroundColor: []\n }\n \n var borderColor = self.ctx.settings.borderColor || '#fff';\n var borderWidth = typeof self.ctx.settings.borderWidth !== 'undefined' ? self.ctx.settings.borderWidth : 5;\n \n pieData.datasets.push(dataset);\n \n for (var i=0; i < self.ctx.data.length; i++) {\n var dataKey = self.ctx.data[i].dataKey;\n var units = dataKey.units && dataKey.units.length ? dataKey.units : self.ctx.units;\n units = units ? (' (' + units + ')') : '';\n pieData.labels.push(dataKey.label + units);\n dataset.data.push(0);\n var hoverBackgroundColor = tinycolor(dataKey.color).lighten(15);\n dataset.backgroundColor.push(dataKey.color);\n dataset.borderColor.push(borderColor);\n dataset.borderWidth.push(borderWidth);\n dataset.hoverBackgroundColor.push(hoverBackgroundColor.toRgbString());\n }\n\n var options = {\n responsive: false,\n maintainAspectRatio: false,\n legend: {\n display: true,\n labels: {\n fontColor: '#666'\n }\n },\n tooltips: {\n callbacks: {\n label: function(tooltipItem, data) {\n var label = data.labels[tooltipItem.index];\n var value = data.datasets[tooltipItem.datasetIndex].data[tooltipItem.index];\n var content = label + ': ' + value;\n var units = self.ctx.settings.units ? self.ctx.settings.units : self.ctx.units;\n if (units) {\n content += ' ' + units;\n } \n return content;\n }\n }\n }\n };\n\n if (self.ctx.settings.legend) {\n options.legend.display = self.ctx.settings.legend.display !== false;\n options.legend.labels.fontColor = self.ctx.settings.legend.labelsFontColor || '#666';\n }\n\n var ctx = $('#pieChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'doughnut',\n data: pieData,\n options: options\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.data.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData.data.length > 0) {\n var decimals;\n if (typeof cellData.dataKey.decimals !== 'undefined' \n && cellData.dataKey.decimals !== null ) {\n decimals = cellData.dataKey.decimals; \n } else {\n decimals = self.ctx.decimals;\n }\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = self.ctx.utils.formatValue(tvPair[1], decimals);\n self.ctx.chart.data.datasets[0].data[i] = parseFloat(value);\n }\n }\n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n self.ctx.chart.resize();\n}\n\n", "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Settings\",\n \"properties\": {\n \"borderWidth\": {\n \"title\": \"Border width\",\n \"type\": \"number\",\n \"default\": 5\n },\n \"borderColor\": {\n \"title\": \"Border color\",\n \"type\": \"string\",\n \"default\": \"#fff\"\n },\n \"legend\": {\n \"title\": \"Legend settings\",\n \"type\": \"object\",\n \"properties\": {\n \"display\": {\n \"title\": \"Display legend\",\n \"type\": \"boolean\",\n \"default\": true\n },\n \"labelsFontColor\": {\n \"title\": \"Labels font color\",\n \"type\": \"string\",\n \"default\": \"#666\"\n }\n }\n }\n },\n \"required\": []\n },\n \"form\": [\n \"borderWidth\", \n {\n \"key\": \"borderColor\",\n \"type\": \"color\"\n }, \n {\n \"key\": \"legend\",\n \"items\": [\n \"legend.display\",\n {\n \"key\": \"legend.labelsFontColor\",\n \"type\": \"color\"\n }\n ]\n }\n ]\n}", "dataKeySettingsSchema": "{}\n", "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"First\",\"color\":\"#26a69a\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Second\",\"color\":\"#f57c00\",\"settings\":{},\"_hash\":0.545701115289893,\"funcBody\":\"var value = (prevValue-20) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+20;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Third\",\"color\":\"#afb42b\",\"settings\":{},\"_hash\":0.2592906835158064,\"funcBody\":\"var value = (prevValue-40) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+40;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fourth\",\"color\":\"#673ab7\",\"settings\":{},\"_hash\":0.12880275585455747,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{\"borderWidth\":5,\"borderColor\":\"#fff\",\"legend\":{\"display\":true,\"labelsFontColor\":\"#666666\"}},\"title\":\"Doughnut - Chart.js\",\"dropShadow\":true,\"enableFullscreen\":true,\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400}}" @@ -91,7 +91,7 @@ ], "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n \n var pieData = {\n labels: [],\n datasets: []\n };\n\n var dataset = {\n data: [],\n backgroundColor: [],\n borderColor: [],\n borderWidth: [],\n hoverBackgroundColor: []\n }\n \n pieData.datasets.push(dataset);\n \n for (var i=0; i < self.ctx.data.length; i++) {\n var dataKey = self.ctx.data[i].dataKey;\n pieData.labels.push(dataKey.label);\n dataset.data.push(0);\n var hoverBackgroundColor = tinycolor(dataKey.color).lighten(15);\n var borderColor = tinycolor(dataKey.color).darken();\n dataset.backgroundColor.push(dataKey.color);\n dataset.borderColor.push('#fff');\n dataset.borderWidth.push(5);\n dataset.hoverBackgroundColor.push(hoverBackgroundColor.toRgbString());\n }\n\n var ctx = $('#pieChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'pie',\n data: pieData,\n options: {\n responsive: false,\n maintainAspectRatio: false\n }\n }); \n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.data.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData.data.length > 0) {\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = tvPair[1];\n self.ctx.chart.data.datasets[0].data[i] = parseFloat(value);\n }\n }\n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n self.ctx.chart.resize();\n}\n", + "controllerScript": "self.onInit = function() {\n \n var pieData = {\n labels: [],\n datasets: []\n };\n\n var dataset = {\n data: [],\n backgroundColor: [],\n borderColor: [],\n borderWidth: [],\n hoverBackgroundColor: []\n }\n \n pieData.datasets.push(dataset);\n \n for (var i=0; i < self.ctx.data.length; i++) {\n var dataKey = self.ctx.data[i].dataKey;\n var units = dataKey.units && dataKey.units.length ? dataKey.units : self.ctx.units;\n units = units ? (' (' + units + ')') : '';\n pieData.labels.push(dataKey.label + units);\n dataset.data.push(0);\n var hoverBackgroundColor = tinycolor(dataKey.color).lighten(15);\n var borderColor = tinycolor(dataKey.color).darken();\n dataset.backgroundColor.push(dataKey.color);\n dataset.borderColor.push('#fff');\n dataset.borderWidth.push(5);\n dataset.hoverBackgroundColor.push(hoverBackgroundColor.toRgbString());\n }\n\n var ctx = $('#pieChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'pie',\n data: pieData,\n options: {\n responsive: false,\n maintainAspectRatio: false\n }\n }); \n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.data.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData.data.length > 0) {\n var decimals;\n if (typeof cellData.dataKey.decimals !== 'undefined' \n && cellData.dataKey.decimals !== null ) {\n decimals = cellData.dataKey.decimals; \n } else {\n decimals = self.ctx.decimals;\n }\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = self.ctx.utils.formatValue(tvPair[1], decimals);\n self.ctx.chart.data.datasets[0].data[i] = parseFloat(value);\n }\n }\n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n self.ctx.chart.resize();\n}\n", "settingsSchema": "{}", "dataKeySettingsSchema": "{}\n", "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"First\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Second\",\"color\":\"#4caf50\",\"settings\":{},\"_hash\":0.545701115289893,\"funcBody\":\"var value = (prevValue-20) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+20;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Third\",\"color\":\"#f44336\",\"settings\":{},\"_hash\":0.2592906835158064,\"funcBody\":\"var value = (prevValue-40) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+40;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fourth\",\"color\":\"#ffc107\",\"settings\":{},\"_hash\":0.12880275585455747,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Pie - Chart.js\"}" @@ -111,7 +111,7 @@ ], "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n var pieData = {\n labels: [],\n datasets: []\n };\n\n var dataset = {\n data: [],\n backgroundColor: [],\n borderColor: [],\n borderWidth: [],\n hoverBackgroundColor: []\n }\n \n pieData.datasets.push(dataset);\n \n for (var i = 0; i < self.ctx.data.length; i++) {\n var dataKey = self.ctx.data[i].dataKey;\n pieData.labels.push(dataKey.label);\n dataset.data.push(0);\n var hoverBackgroundColor = tinycolor(dataKey.color).lighten(15);\n var borderColor = tinycolor(dataKey.color).darken();\n dataset.backgroundColor.push(dataKey.color);\n dataset.borderColor.push('#fff');\n dataset.borderWidth.push(5);\n dataset.hoverBackgroundColor.push(hoverBackgroundColor.toRgbString());\n }\n\n var ctx = $('#pieChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'polarArea',\n data: pieData,\n options: {\n responsive: false,\n maintainAspectRatio: false\n }\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.data.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData.data.length > 0) {\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = tvPair[1];\n self.ctx.chart.data.datasets[0].data[i] = parseFloat(value);\n }\n }\n \n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n if (self.ctx.height >= 70) {\n try {\n self.ctx.chart.resize();\n } catch (e) {}\n }\n}\n", + "controllerScript": "self.onInit = function() {\n var pieData = {\n labels: [],\n datasets: []\n };\n\n var dataset = {\n data: [],\n backgroundColor: [],\n borderColor: [],\n borderWidth: [],\n hoverBackgroundColor: []\n }\n \n pieData.datasets.push(dataset);\n \n for (var i = 0; i < self.ctx.data.length; i++) {\n var dataKey = self.ctx.data[i].dataKey;\n var units = dataKey.units && dataKey.units.length ? dataKey.units : self.ctx.units;\n units = units ? (' (' + units + ')') : '';\n pieData.labels.push(dataKey.label + units);\n dataset.data.push(0);\n var hoverBackgroundColor = tinycolor(dataKey.color).lighten(15);\n var borderColor = tinycolor(dataKey.color).darken();\n dataset.backgroundColor.push(dataKey.color);\n dataset.borderColor.push('#fff');\n dataset.borderWidth.push(5);\n dataset.hoverBackgroundColor.push(hoverBackgroundColor.toRgbString());\n }\n\n var ctx = $('#pieChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'polarArea',\n data: pieData,\n options: {\n responsive: false,\n maintainAspectRatio: false\n }\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.data.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData.data.length > 0) {\n var decimals;\n if (typeof cellData.dataKey.decimals !== 'undefined' \n && cellData.dataKey.decimals !== null ) {\n decimals = cellData.dataKey.decimals; \n } else {\n decimals = self.ctx.decimals;\n }\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = self.ctx.utils.formatValue(tvPair[1], decimals);\n self.ctx.chart.data.datasets[0].data[i] = parseFloat(value);\n }\n }\n \n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n if (self.ctx.height >= 70) {\n try {\n self.ctx.chart.resize();\n } catch (e) {}\n }\n}\n", "settingsSchema": "{}", "dataKeySettingsSchema": "{}\n", "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"First\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Second\",\"color\":\"#4caf50\",\"settings\":{},\"_hash\":0.545701115289893,\"funcBody\":\"var value = (prevValue-20) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+20;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Third\",\"color\":\"#f44336\",\"settings\":{},\"_hash\":0.2592906835158064,\"funcBody\":\"var value = (prevValue-40) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+40;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fourth\",\"color\":\"#ffc107\",\"settings\":{},\"_hash\":0.12880275585455747,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fifth\",\"color\":\"#607d8b\",\"settings\":{},\"_hash\":0.2074391823443591,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Polar Area - Chart.js\"}" @@ -131,7 +131,7 @@ ], "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n var barData = {\n labels: [],\n datasets: []\n };\n\n var backgroundColor = tinycolor(self.ctx.data[0].dataKey.color);\n backgroundColor.setAlpha(0.2);\n var borderColor = tinycolor(self.ctx.data[0].dataKey.color);\n borderColor.setAlpha(1);\n var dataset = {\n label: self.ctx.datasources[0].name,\n data: [],\n backgroundColor: backgroundColor.toRgbString(),\n borderColor: borderColor.toRgbString(),\n pointBackgroundColor: borderColor.toRgbString(),\n pointBorderColor: borderColor.darken().toRgbString(),\n borderWidth: 1\n }\n \n barData.datasets.push(dataset);\n \n for (var i = 0; i < self.ctx.data.length; i++) {\n var dataKey = self.ctx.data[i].dataKey;\n barData.labels.push(dataKey.label);\n dataset.data.push(0);\n }\n\n var ctx = $('#radarChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'radar',\n data: barData,\n options: {\n responsive: false,\n maintainAspectRatio: false\n }\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.data.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData.data.length > 0) {\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = tvPair[1];\n self.ctx.chart.data.datasets[0].data[i] = parseFloat(value);\n }\n } \n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n if (self.ctx.height >= 70) {\n self.ctx.chart.resize();\n }\n}\n", + "controllerScript": "self.onInit = function() {\n var barData = {\n labels: [],\n datasets: []\n };\n\n var backgroundColor = tinycolor(self.ctx.data[0].dataKey.color);\n backgroundColor.setAlpha(0.2);\n var borderColor = tinycolor(self.ctx.data[0].dataKey.color);\n borderColor.setAlpha(1);\n var dataset = {\n label: self.ctx.datasources[0].name,\n data: [],\n backgroundColor: backgroundColor.toRgbString(),\n borderColor: borderColor.toRgbString(),\n pointBackgroundColor: borderColor.toRgbString(),\n pointBorderColor: borderColor.darken().toRgbString(),\n borderWidth: 1\n }\n \n barData.datasets.push(dataset);\n \n for (var i = 0; i < self.ctx.data.length; i++) {\n var dataKey = self.ctx.data[i].dataKey;\n var units = dataKey.units && dataKey.units.length ? dataKey.units : self.ctx.units;\n units = units ? (' (' + units + ')') : '';\n barData.labels.push(dataKey.label + units);\n dataset.data.push(0);\n }\n\n var ctx = $('#radarChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'radar',\n data: barData,\n options: {\n responsive: false,\n maintainAspectRatio: false\n }\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.data.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData.data.length > 0) {\n var decimals;\n if (typeof cellData.dataKey.decimals !== 'undefined' \n && cellData.dataKey.decimals !== null ) {\n decimals = cellData.dataKey.decimals; \n } else {\n decimals = self.ctx.decimals;\n }\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = self.ctx.utils.formatValue(tvPair[1], decimals);\n self.ctx.chart.data.datasets[0].data[i] = parseFloat(value);\n }\n } \n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n if (self.ctx.height >= 70) {\n self.ctx.chart.resize();\n }\n}\n", "settingsSchema": "{}", "dataKeySettingsSchema": "{}\n", "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"First\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Second\",\"color\":\"#4caf50\",\"settings\":{},\"_hash\":0.545701115289893,\"funcBody\":\"var value = (prevValue-20) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+20;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Third\",\"color\":\"#f44336\",\"settings\":{},\"_hash\":0.2592906835158064,\"funcBody\":\"var value = (prevValue-40) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+40;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fourth\",\"color\":\"#ffc107\",\"settings\":{},\"_hash\":0.12880275585455747,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Radar - Chart.js\"}" diff --git a/application/src/main/data/json/system/widget_bundles/input_widgets.json b/application/src/main/data/json/system/widget_bundles/input_widgets.json index 1d5b28cba6..4ed3c2fab6 100644 --- a/application/src/main/data/json/system/widget_bundles/input_widgets.json +++ b/application/src/main/data/json/system/widget_bundles/input_widgets.json @@ -32,8 +32,8 @@ "templateHtml": "\n", "templateCss": ".tb-toast {\n min-width: 0;\n font-size: 14px !important;\n}", "controllerScript": "self.onInit = function() {\r\n}\r\n\r\nself.onDataUpdated = function() {\r\n self.ctx.$scope.multipleInputWidget.onDataUpdated();\r\n}\r\n", - "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"MultipleInput\",\n \"properties\": {\n \"widgetTitle\": {\n \"title\": \"Widget title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"showActionButtons\":{\n \"title\":\"Show action buttons\",\n \"type\":\"boolean\",\n \"default\": true\n },\n \"updateAllValues\": {\n \"title\":\"Update all values, not only modified (only if action buttons are visible)\",\n \"type\":\"boolean\",\n \"default\": false\n },\n \"showResultMessage\":{\n \"title\":\"Show result message\",\n \"type\":\"boolean\",\n \"default\": true\n },\n \"showGroupTitle\": {\n \"title\":\"Show title for group of fields, related to different entities\",\n \"type\":\"boolean\",\n \"default\": false\n },\n \"groupTitle\": {\n \"title\": \"Group title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"fieldsAlignment\": {\n \"title\": \"Fields alignment\",\n \"type\": \"string\",\n \"default\": \"row\"\n },\n \"fieldsInRow\": {\n \"title\": \"Number of fields in the row\",\n \"type\": \"number\",\n \"default\": \"2\"\n }\n },\n \"required\": []\n },\n \"form\": [\n \"widgetTitle\",\n \"showActionButtons\",\n \"updateAllValues\",\n \"showResultMessage\",\n \"showGroupTitle\",\n \"groupTitle\",\n {\n \"key\": \"fieldsAlignment\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"row\",\n \"label\": \"Row (default)\"\n },\n {\n \"value\": \"column\",\n \"label\": \"Column\"\n }\n ]\n },\n \"fieldsInRow\"\n ]\n}", - "dataKeySettingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"DataKeySettings\",\n \"properties\": {\n \"dataKeyType\": {\n \"title\": \"Datakey type\",\n \"type\": \"string\",\n \"default\": \"server\"\n },\n \"dataKeyValueType\": {\n \"title\": \"Datakey value type\",\n \"type\": \"string\",\n \"default\": \"string\"\n },\n \"required\": {\n \"title\": \"Value is required\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"isEditable\": {\n \"title\": \"Ability to edit attribute\",\n \"type\": \"string\",\n \"default\": \"editable\"\n },\n \"disabledOnDataKey\": {\n \"title\": \"Disable on false value of another datakey (specify datakey name)\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"dataKeyHidden\": {\n \"title\": \"Hide input field\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"step\": {\n \"title\": \"Step interval between values (only for numbers)\",\n \"type\": \"number\",\n \"default\": \"1\"\n },\n \"requiredErrorMessage\": {\n \"title\": \"'Required' error message\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"icon\": {\n \"title\": \"Icon to show before input cell\",\n \"type\": \"string\",\n \"default\": \"\"\n }\n },\n \"required\": []\n },\n \"form\": [\n {\n \"key\": \"dataKeyType\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"server\",\n \"label\": \"Server attribute (default)\"\n },\n {\n \"value\": \"shared\",\n \"label\": \"Shared attribute\"\n },\n {\n \"value\": \"timeseries\",\n \"label\": \"Timeseries\"\n }\n ]\n },\n {\n \"key\": \"dataKeyValueType\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"string\",\n \"label\": \"String\"\n },\n {\n \"value\": \"double\",\n \"label\": \"Double\"\n },\n {\n \"value\": \"integer\",\n \"label\": \"Integer\"\n },\n {\n \"value\": \"booleanCheckbox\",\n \"label\": \"Boolean (Checkbox)\"\n },\n {\n \"value\": \"booleanSwitch\",\n \"label\": \"Boolean (Switch)\"\n },\n {\n \"value\": \"dateTime\",\n \"label\": \"Date & Time\"\n },\n {\n \"value\": \"date\",\n \"label\": \"Date\"\n },\n {\n \"value\": \"time\",\n \"label\": \"Time\"\n }\n ]\n },\n \"required\",\n {\n \"key\": \"isEditable\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"editable\",\n \"label\": \"Editable (default)\"\n },\n {\n \"value\": \"disabled\",\n \"label\": \"Disabled\"\n },\n {\n \"value\": \"readonly\",\n \"label\": \"Read-only\"\n }\n ]\n },\n \"disabledOnDataKey\",\n \"dataKeyHidden\",\n \"step\",\n \"requiredErrorMessage\",\n\t\t{\n \t\t\"key\": \"icon\",\n\t\t\t\"type\": \"icon\"\n\t\t}\n ]\n}\n", + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"MultipleInput\",\n \"properties\": {\n \"widgetTitle\": {\n \"title\": \"Widget title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"showActionButtons\":{\n \"title\":\"Show action buttons\",\n \"type\":\"boolean\",\n \"default\": true\n },\n \"updateAllValues\": {\n \"title\":\"Update all values, not only modified\",\n \"type\":\"boolean\",\n \"default\": false\n },\n \"saveButtonLabel\": {\n \"title\": \"'SAVE' button label\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"resetButtonLabel\": {\n \"title\": \"'UNDO' button label\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"showResultMessage\":{\n \"title\":\"Show result message\",\n \"type\":\"boolean\",\n \"default\": true\n },\n \"showGroupTitle\": {\n \"title\":\"Show title for group of fields, related to different entities\",\n \"type\":\"boolean\",\n \"default\": false\n },\n \"groupTitle\": {\n \"title\": \"Group title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"fieldsAlignment\": {\n \"title\": \"Fields alignment\",\n \"type\": \"string\",\n \"default\": \"row\"\n },\n \"fieldsInRow\": {\n \"title\": \"Number of fields in the row\",\n \"type\": \"number\",\n \"default\": \"2\"\n }\n },\n \"required\": []\n },\n \"form\": [\n \"widgetTitle\",\n \"showActionButtons\",\n {\n \"key\": \"updateAllValues\",\n \"condition\": \"model.showActionButtons === true\"\n },\n {\n \"key\": \"saveButtonLabel\",\n \"condition\": \"model.showActionButtons === true\"\n },\n {\n \"key\": \"resetButtonLabel\",\n \"condition\": \"model.showActionButtons === true\"\n },\n \"showResultMessage\",\n \"showGroupTitle\",\n \"groupTitle\",\n {\n \"key\": \"fieldsAlignment\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"row\",\n \"label\": \"Row (default)\"\n },\n {\n \"value\": \"column\",\n \"label\": \"Column\"\n }\n ]\n },\n {\n \"key\": \"fieldsInRow\",\n \"condition\": \"model.fieldsAlignment === 'row'\"\n }\n ]\n}", + "dataKeySettingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"DataKeySettings\",\n \"properties\": {\n \"dataKeyType\": {\n \"title\": \"Datakey type\",\n \"type\": \"string\",\n \"default\": \"server\"\n },\n \"dataKeyValueType\": {\n \"title\": \"Datakey value type\",\n \"type\": \"string\",\n \"default\": \"string\"\n },\n \"step\": {\n \"title\": \"Step interval between values\",\n \"type\": \"number\",\n \"default\": \"1\"\n },\n \"minValue\": {\n \"title\": \"Minimum value\",\n \"type\": \"number\"\n },\n \"maxValue\": {\n \"title\": \"Maximum value\",\n \"type\": \"number\"\n },\n \"required\": {\n \"title\": \"Value is required\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"requiredErrorMessage\": {\n \"title\": \"'Required' error message\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"minValueErrorMessage\": {\n \"title\": \"'Min Value' error message\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"maxValueErrorMessage\": {\n \"title\": \"'Max Value' error message\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"invalidDateErrorMessage\": {\n \"title\": \"'Invalid Date' error message\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"isEditable\": {\n \"title\": \"Ability to edit attribute\",\n \"type\": \"string\",\n \"default\": \"editable\"\n },\n \"disabledOnDataKey\": {\n \"title\": \"Disable on false value of another datakey (specify datakey name)\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"dataKeyHidden\": {\n \"title\": \"Hide input field\",\n \"type\": \"boolean\",\n \"default\": false\n },\n \"icon\": {\n \"title\": \"Icon to show before input cell\",\n \"type\": \"string\",\n \"default\": \"\"\n }\n },\n \"required\": []\n },\n \"form\": [\n {\n \"key\": \"dataKeyType\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"server\",\n \"label\": \"Server attribute (default)\"\n },\n {\n \"value\": \"shared\",\n \"label\": \"Shared attribute\"\n },\n {\n \"value\": \"timeseries\",\n \"label\": \"Timeseries\"\n }\n ]\n },\n {\n \"key\": \"dataKeyValueType\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"string\",\n \"label\": \"String\"\n },\n {\n \"value\": \"double\",\n \"label\": \"Double\"\n },\n {\n \"value\": \"integer\",\n \"label\": \"Integer\"\n },\n {\n \"value\": \"booleanCheckbox\",\n \"label\": \"Boolean (Checkbox)\"\n },\n {\n \"value\": \"booleanSwitch\",\n \"label\": \"Boolean (Switch)\"\n },\n {\n \"value\": \"dateTime\",\n \"label\": \"Date & Time\"\n },\n {\n \"value\": \"date\",\n \"label\": \"Date\"\n },\n {\n \"value\": \"time\",\n \"label\": \"Time\"\n }\n ]\n },\n {\n \"key\": \"step\",\n \"condition\": \"model.dataKeyValueType === 'double' || model.dataKeyValueType === 'integer'\"\n },\n {\n \"key\": \"minValue\",\n \"condition\": \"model.dataKeyValueType === 'double' || model.dataKeyValueType === 'integer'\"\n },\n {\n \"key\": \"maxValue\",\n \"condition\": \"model.dataKeyValueType === 'double' || model.dataKeyValueType === 'integer'\"\n },\n \"required\",\n {\n \"key\": \"requiredErrorMessage\",\n \"condition\": \"model.required === true\"\n },\n {\n \"key\": \"invalidDateErrorMessage\",\n \"condition\": \"model.dataKeyValueType === 'dateTime' || model.dataKeyValueType === 'date' || model.dataKeyValueType === 'time'\"\n },\n {\n \"key\": \"minValueErrorMessage\",\n \"condition\": \"model.dataKeyValueType === 'double' || model.dataKeyValueType === 'integer'\"\n },\n {\n \"key\": \"maxValueErrorMessage\",\n \"condition\": \"model.dataKeyValueType === 'double' || model.dataKeyValueType === 'integer'\"\n },\n {\n \"key\": \"isEditable\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"editable\",\n \"label\": \"Editable (default)\"\n },\n {\n \"value\": \"disabled\",\n \"label\": \"Disabled\"\n },\n {\n \"value\": \"readonly\",\n \"label\": \"Read-only\"\n }\n ]\n },\n \"disabledOnDataKey\",\n \"dataKeyHidden\",\n\t\t{\n \t\t\"key\": \"icon\",\n\t\t\t\"type\": \"icon\"\n\t\t}\n ]\n}\n", "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Sin\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.23592248334107624,\"funcBody\":\"return Math.round(1000*Math.sin(time/5000));\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Update Multiple Attributes\",\"dropShadow\":true,\"enableFullscreen\":false,\"enableDataExport\":false,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"showLegend\":false,\"actions\":{}}" } }, @@ -391,16 +391,16 @@ }, { "alias": "web_camera_input", - "name": "Web Camera Input", + "name": "Photo camera input", "descriptor": { "type": "latest", "sizeX": 7.5, "sizeY": 3, "resources": [], - "templateHtml": "\n", + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.webCameraInputWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true\n }\n}\n\nself.onDestroy = function() {\n}\n", - "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Web Camera\",\n \"properties\": {\n \"widgetTitle\": {\n \"title\": \"Widget title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"imageFormat\": {\n \"title\": \"Image Format\",\n \"type\": \"string\",\n \"default\": \"image/png\"\n },\n \"imageQuality\":{\n \"title\":\"Image quality that use lossy compression such as jpeg and webp\",\n \"type\":\"number\",\n \"default\": 0.92,\n \"min\": 0,\n \"max\": 1\n },\n \"maxWidth\": {\n \"title\": \"The maximal image width\",\n \"type\": \"number\",\n \"default\": 640\n }, \n \"maxHeight\": {\n \"title\": \"The maximal image heigth\",\n \"type\": \"number\",\n \"default\": 480\n }\n },\n \"required\": []\n },\n \"form\": [\n \"widgetTitle\",\n {\n \"key\": \"imageFormat\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"image/jpeg\",\n \"label\": \"JPEG\"\n },\n {\n \"value\": \"image/png\",\n \"label\": \"PNG\"\n },\n {\n \"value\": \"image/webp\",\n \"label\": \"WEBP\"\n }\n ]\n },\n \"imageQuality\",\n \"maxWidth\",\n \"maxHeight\"\n ]\n}", + "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.photoCameraInputWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true\n }\n}\n\nself.onDestroy = function() {\n}\n", + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Photo Camera\",\n \"properties\": {\n \"widgetTitle\": {\n \"title\": \"Widget title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"imageFormat\": {\n \"title\": \"Image Format\",\n \"type\": \"string\",\n \"default\": \"image/png\"\n },\n \"imageQuality\":{\n \"title\":\"Image quality that use lossy compression such as jpeg and webp\",\n \"type\":\"number\",\n \"default\": 0.92,\n \"min\": 0,\n \"max\": 1\n },\n \"maxWidth\": {\n \"title\": \"The maximal image width\",\n \"type\": \"number\",\n \"default\": 640\n }, \n \"maxHeight\": {\n \"title\": \"The maximal image heigth\",\n \"type\": \"number\",\n \"default\": 480\n }\n },\n \"required\": []\n },\n \"form\": [\n \"widgetTitle\",\n {\n \"key\": \"imageFormat\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"image/jpeg\",\n \"label\": \"JPEG\"\n },\n {\n \"value\": \"image/png\",\n \"label\": \"PNG\"\n },\n {\n \"value\": \"image/webp\",\n \"label\": \"WEBP\"\n }\n ]\n },\n \"imageQuality\",\n \"maxWidth\",\n \"maxHeight\"\n ]\n}", "dataKeySettingsSchema": "{}\n", "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Web Camera Input\",\"showTitleIcon\":false,\"titleIcon\":\"more_horiz\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"titleTooltip\":\"\",\"dropShadow\":true,\"enableFullscreen\":false,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"displayTimewindow\":true,\"showLegend\":false,\"actions\":{}}" } diff --git a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json index 3d076ff812..da9a95b423 100644 --- a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json +++ b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json @@ -94,7 +94,8 @@ "name": "Device Profile Node", "debugMode": false, "configuration": { - "persistAlarmRulesState": false + "persistAlarmRulesState": false, + "fetchAlarmRulesStateOnStart": false } } ], diff --git a/application/src/main/data/upgrade/3.1.1/schema_update_before.sql b/application/src/main/data/upgrade/3.1.1/schema_update_before.sql index c1591e7831..5f30f135e1 100644 --- a/application/src/main/data/upgrade/3.1.1/schema_update_before.sql +++ b/application/src/main/data/upgrade/3.1.1/schema_update_before.sql @@ -14,19 +14,92 @@ -- limitations under the License. -- +CREATE TABLE IF NOT EXISTS oauth2_client_registration_info ( + id uuid NOT NULL CONSTRAINT oauth2_client_registration_info_pkey PRIMARY KEY, + enabled boolean, + created_time bigint NOT NULL, + additional_info varchar, + client_id varchar(255), + client_secret varchar(255), + authorization_uri varchar(255), + token_uri varchar(255), + scope varchar(255), + user_info_uri varchar(255), + user_name_attribute_name varchar(255), + jwk_set_uri varchar(255), + client_authentication_method varchar(255), + login_button_label varchar(255), + login_button_icon varchar(255), + allow_user_creation boolean, + activate_user boolean, + type varchar(31), + basic_email_attribute_key varchar(31), + basic_first_name_attribute_key varchar(31), + basic_last_name_attribute_key varchar(31), + basic_tenant_name_strategy varchar(31), + basic_tenant_name_pattern varchar(255), + basic_customer_name_pattern varchar(255), + basic_default_dashboard_name varchar(255), + basic_always_full_screen boolean, + custom_url varchar(255), + custom_username varchar(255), + custom_password varchar(255), + custom_send_token boolean +); + +CREATE TABLE IF NOT EXISTS oauth2_client_registration ( + id uuid NOT NULL CONSTRAINT oauth2_client_registration_pkey PRIMARY KEY, + created_time bigint NOT NULL, + domain_name varchar(255), + domain_scheme varchar(31), + client_registration_info_id uuid +); + +CREATE TABLE IF NOT EXISTS oauth2_client_registration_template ( + id uuid NOT NULL CONSTRAINT oauth2_client_registration_template_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + provider_id varchar(255), + authorization_uri varchar(255), + token_uri varchar(255), + scope varchar(255), + user_info_uri varchar(255), + user_name_attribute_name varchar(255), + jwk_set_uri varchar(255), + client_authentication_method varchar(255), + type varchar(31), + basic_email_attribute_key varchar(31), + basic_first_name_attribute_key varchar(31), + basic_last_name_attribute_key varchar(31), + basic_tenant_name_strategy varchar(31), + basic_tenant_name_pattern varchar(255), + basic_customer_name_pattern varchar(255), + basic_default_dashboard_name varchar(255), + basic_always_full_screen boolean, + comment varchar, + login_button_icon varchar(255), + login_button_label varchar(255), + help_link varchar(255), + CONSTRAINT oauth2_template_provider_id_unq_key UNIQUE (provider_id) +); + CREATE TABLE IF NOT EXISTS device_profile ( id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, created_time bigint NOT NULL, name varchar(255), type varchar(255), transport_type varchar(255), + provision_type varchar(255), profile_data jsonb, description varchar, search_text varchar(255), is_default boolean, tenant_id uuid, default_rule_chain_id uuid, + default_queue_name varchar(255), + provision_device_key varchar, CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain(id) ); diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 50e837769a..9b108e9b8a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -32,6 +32,8 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.SmsService; +import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.common.data.DataConstants; @@ -67,6 +69,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -74,12 +78,14 @@ import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; import org.thingsboard.server.service.mail.MailExecutorService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.script.JsExecutorService; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.session.DeviceSessionCacheService; +import org.thingsboard.server.service.sms.SmsExecutorService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -107,6 +113,14 @@ public class ActorSystemContext { return debugPerTenantLimits; } + @Autowired + @Getter + private TbApiUsageStateService apiUsageStateService; + + @Autowired + @Getter + private TbApiUsageClient apiUsageClient; + @Autowired @Getter @Setter @@ -129,6 +143,10 @@ public class ActorSystemContext { @Getter private DeviceService deviceService; + @Autowired + @Getter + private TbTenantProfileCache tenantProfileCache; + @Autowired @Getter private TbDeviceProfileCache deviceProfileCache; @@ -216,6 +234,10 @@ public class ActorSystemContext { @Getter private MailExecutorService mailExecutor; + @Autowired + @Getter + private SmsExecutorService smsExecutor; + @Autowired @Getter private DbCallbackExecutorService dbCallbackExecutor; @@ -232,6 +254,14 @@ public class ActorSystemContext { @Getter private MailService mailService; + @Autowired + @Getter + private SmsService smsService; + + @Autowired + @Getter + private SmsSenderFactory smsSenderFactory; + @Autowired @Getter private ClaimDevicesService claimDevicesService; @@ -325,6 +355,10 @@ public class ActorSystemContext { @Getter private boolean allowSystemMailService; + @Value("${actors.rule.allow_system_sms_service}") + @Getter + private boolean allowSystemSmsService; + @Value("${transport.sessions.inactivity_timeout}") @Getter private long sessionInactivityTimeout; @@ -553,7 +587,11 @@ public class ActorSystemContext { public void scheduleMsgWithDelay(TbActorRef ctx, TbActorMsg msg, long delayInMs) { log.debug("Scheduling msg {} with delay {} ms", msg, delayInMs); - getScheduler().schedule(() -> ctx.tell(msg), delayInMs, TimeUnit.MILLISECONDS); + if (delayInMs > 0) { + getScheduler().schedule(() -> ctx.tell(msg), delayInMs, TimeUnit.MILLISECONDS); + } else { + ctx.tell(msg); + } } } diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 16beb3a045..ba8654854c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -39,8 +39,8 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import java.util.HashSet; @@ -50,13 +50,14 @@ import java.util.Set; @Slf4j public class AppActor extends ContextAwareActor { - private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); + private final TbTenantProfileCache tenantProfileCache; private final TenantService tenantService; private final Set deletedTenants; private boolean ruleChainsInitialized; private AppActor(ActorSystemContext systemContext) { super(systemContext); + this.tenantProfileCache = systemContext.getTenantProfileCache(); this.tenantService = systemContext.getTenantService(); this.deletedTenants = new HashSet<>(); } @@ -117,8 +118,7 @@ public class AppActor extends ContextAwareActor { boolean isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); boolean isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); for (Tenant tenant : tenantIterator) { - // TODO: Tenant Profile from cache - TenantProfile tenantProfile = systemContext.getTenantProfileService().findTenantProfileById(TenantId.SYS_TENANT_ID, tenant.getTenantProfileId()); + TenantProfile tenantProfile = tenantProfileCache.get(tenant.getTenantProfileId()); if (isCore || (isRuleEngine && !tenantProfile.isIsolatedTbRuleEngine())) { log.debug("[{}] Creating tenant actor", tenant.getId()); getOrCreateTenantActor(tenant.getId()); @@ -133,7 +133,7 @@ public class AppActor extends ContextAwareActor { } private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { - if (SYSTEM_TENANT.equals(msg.getTenantId())) { + if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { msg.getTbMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { if (!deletedTenants.contains(msg.getTenantId())) { @@ -146,15 +146,20 @@ public class AppActor extends ContextAwareActor { private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { TbActorRef target = null; - if (SYSTEM_TENANT.equals(msg.getTenantId())) { - log.warn("Message has system tenant id: {}", msg); + if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { + if (!EntityType.TENANT_PROFILE.equals(msg.getEntityId().getEntityType())) { + log.warn("Message has system tenant id: {}", msg); + } } else { - if (msg.getEntityId().getEntityType() == EntityType.TENANT - && msg.getEvent() == ComponentLifecycleEvent.DELETED) { - log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); + if (EntityType.TENANT.equals(msg.getEntityId().getEntityType())) { TenantId tenantId = new TenantId(msg.getEntityId().getId()); - deletedTenants.add(tenantId); - ctx.stop(new TbEntityActorId(tenantId)); + if (msg.getEvent() == ComponentLifecycleEvent.DELETED) { + log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); + deletedTenants.add(tenantId); + ctx.stop(new TbEntityActorId(tenantId)); + } else { + target = getOrCreateTenantActor(msg.getTenantId()); + } } else { target = getOrCreateTenantActor(msg.getTenantId()); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index ab65d99cf4..5a1951dad6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.channel.EventLoopGroup; import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.util.StringUtils; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; @@ -27,15 +28,21 @@ import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbRelationTypes; +import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; @@ -47,6 +54,7 @@ import org.thingsboard.server.common.data.rule.RuleNodeState; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.asset.AssetService; @@ -72,6 +80,7 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import java.util.Collections; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; /** @@ -185,6 +194,9 @@ class DefaultTbContext implements TbContext { } private TopicPartitionInfo resolvePartition(TbMsg tbMsg, String queueName) { + if (StringUtils.isEmpty(queueName)) { + queueName = ServiceQueue.MAIN; + } return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); } @@ -253,26 +265,26 @@ class DefaultTbContext implements TbContext { } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { - return entityCreatedMsg(customer, customer.getId(), ruleNodeId); + return entityActionMsg(customer, customer.getId(), ruleNodeId, DataConstants.ENTITY_CREATED); } public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { - return entityCreatedMsg(device, device.getId(), ruleNodeId); + return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED); } public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { - return entityCreatedMsg(asset, asset.getId(), ruleNodeId); + return entityActionMsg(asset, asset.getId(), ruleNodeId, DataConstants.ENTITY_CREATED); } - public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) { - return entityCreatedMsg(alarm, alarm.getId(), ruleNodeId); + public TbMsg alarmActionMsg(Alarm alarm, RuleNodeId ruleNodeId, String action) { + return entityActionMsg(alarm, alarm.getId(), ruleNodeId, action); } - public TbMsg entityCreatedMsg(E entity, I id, RuleNodeId ruleNodeId) { + public TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action) { try { - return TbMsg.newMsg(DataConstants.ENTITY_CREATED, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); + return TbMsg.newMsg(action, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " created msg: " + e); + throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " " + action + " msg: " + e); } } @@ -296,6 +308,11 @@ class DefaultTbContext implements TbContext { return mainCtx.getMailExecutor(); } + @Override + public ListeningExecutor getSmsExecutor() { + return mainCtx.getSmsExecutor(); + } + @Override public ListeningExecutor getDbCallbackExecutor() { return mainCtx.getDbCallbackExecutor(); @@ -308,7 +325,7 @@ class DefaultTbContext implements TbContext { @Override public ScriptEngine createJsScriptEngine(String script, String... argNames) { - return new RuleNodeJsScriptEngine(mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames); + return new RuleNodeJsScriptEngine(getTenantId(), mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames); } @Override @@ -431,6 +448,20 @@ class DefaultTbContext implements TbContext { } } + @Override + public SmsService getSmsService() { + if (mainCtx.isAllowSystemSmsService()) { + return mainCtx.getSmsService(); + } else { + throw new RuntimeException("Access to System SMS Service is forbidden!"); + } + } + + @Override + public SmsSenderFactory getSmsSenderFactory() { + return mainCtx.getSmsSenderFactory(); + } + @Override public RuleEngineRpcService getRpcService() { return mainCtx.getTbRuleEngineDeviceRpcService(); @@ -476,6 +507,43 @@ class DefaultTbContext implements TbContext { return mainCtx.getRuleNodeStateService().save(getTenantId(), state); } + @Override + public void clearRuleNodeStates() { + if (log.isDebugEnabled()) { + log.debug("[{}][{}] Going to clear rule node states", getTenantId(), getSelfId()); + } + mainCtx.getRuleNodeStateService().removeByRuleNodeId(getTenantId(), getSelfId()); + } + + @Override + public void removeRuleNodeStateForEntity(EntityId entityId) { + if (log.isDebugEnabled()) { + log.debug("[{}][{}][{}] Remove Rule Node State for entity.", getTenantId(), getSelfId(), entityId); + } + mainCtx.getRuleNodeStateService().removeByRuleNodeIdAndEntityId(getTenantId(), getSelfId(), entityId); + } + + @Override + public void addTenantProfileListener(Consumer listener) { + mainCtx.getTenantProfileCache().addListener(getTenantId(), getSelfId(), listener); + } + + @Override + public void addDeviceProfileListeners(Consumer profileListener, BiConsumer deviceListener) { + mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), profileListener, deviceListener); + } + + @Override + public void removeListeners() { + mainCtx.getDeviceProfileCache().removeListener(getTenantId(), getSelfId()); + mainCtx.getTenantProfileCache().removeListener(getTenantId(), getSelfId()); + } + + @Override + public TenantProfile getTenantProfile() { + return mainCtx.getTenantProfileCache().get(getTenantId()); + } + private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("ruleNodeId", ruleNodeId.toString()); diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index b5c7531293..9feac853ca 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -23,6 +23,7 @@ import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.ComponentMsgProcessor; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -47,6 +48,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; import org.thingsboard.server.service.queue.TbClusterService; import java.util.ArrayList; @@ -69,15 +71,16 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor> nodeRoutes; private final RuleChainService service; private final TbClusterService clusterService; + private final TbApiUsageClient apiUsageClient; private String ruleChainName; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; - RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext - , TbActorRef parent, TbActorRef self) { + RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext, TbActorRef parent, TbActorRef self) { super(systemContext, tenantId, ruleChain.getId()); + this.apiUsageClient = systemContext.getApiUsageClient(); this.ruleChainName = ruleChain.getName(); this.parent = parent; this.self = self; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java index 3432950511..5cea8b5d9a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java @@ -64,6 +64,12 @@ public abstract class RuleChainManagerActor extends ContextAwareActor { } } + protected void destroyRuleChains() { + for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChainsByType(tenantId, RuleChainType.CORE, link), ContextAwareActor.ENTITY_PACK_LIMIT)) { + ctx.stop(new TbEntityActorId(ruleChain.getId())); + } + } + protected void visit(RuleChain entity, TbActorRef actorRef) { if (entity != null && entity.isRoot() && entity.getType().equals(RuleChainType.CORE)) { rootChain = entity; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java index db55ff8edf..95d21090bd 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java @@ -21,13 +21,18 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.shared.ComponentMsgProcessor; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; +import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.RuleNodeException; import org.thingsboard.server.common.msg.queue.RuleNodeInfo; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; /** * @author Andrew Shvayka @@ -36,6 +41,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor extends Abstract this.entityId = id; } + protected TenantProfileConfiguration getTenantProfileConfiguration() { + return systemContext.getTenantProfileCache().get(tenantId).getProfileData().getConfiguration(); + } + public abstract String getComponentName(); public abstract void start(TbActorCtx context) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index abe07f0e51..3240726065 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -29,6 +29,7 @@ import org.thingsboard.server.actors.device.DeviceActorCreator; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; @@ -62,6 +63,7 @@ public class TenantActor extends RuleChainManagerActor { private boolean isRuleEngineForCurrentTenant; private boolean isCore; + private ApiUsageState apiUsageState; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext, tenantId); @@ -79,21 +81,24 @@ public class TenantActor extends RuleChainManagerActor { cantFindTenant = true; log.info("[{}] Started tenant actor for missing tenant.", tenantId); } else { + apiUsageState = new ApiUsageState(systemContext.getApiUsageStateService().getApiUsageState(tenant.getId())); + // This Service may be started for specific tenant only. Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); - // TODO: Tenant Profile from cache - - TenantProfile tenantProfile = systemContext.getTenantProfileService().findTenantProfileById(tenantId, tenant.getTenantProfileId()); + TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId()); - isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); if (isRuleEngineForCurrentTenant) { try { if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenantProfile.isIsolatedTbRuleEngine())) { - log.info("[{}] Going to init rule chains", tenantId); - initRuleChains(); + if (apiUsageState.isReExecEnabled()) { + log.info("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + log.info("[{}] Skip init of the rule chains due to API limits", tenantId); + } } else { isRuleEngineForCurrentTenant = false; } @@ -105,8 +110,6 @@ public class TenantActor extends RuleChainManagerActor { } } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); -// TODO: throw this in 3.1? -// throw new TbActorException("Failed to init actor", e); } } @@ -122,7 +125,7 @@ public class TenantActor extends RuleChainManagerActor { if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; queueMsg.getTbMsg().getCallback().onSuccess(); - } else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)){ + } else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)) { TransportToDeviceActorMsgWrapper transportMsg = (TransportToDeviceActorMsgWrapper) msg; transportMsg.getCallback().onSuccess(); } @@ -180,26 +183,33 @@ public class TenantActor extends RuleChainManagerActor { return; } TbMsg tbMsg = msg.getTbMsg(); - if (tbMsg.getRuleChainId() == null) { - if (getRootChainActor() != null) { - getRootChainActor().tell(msg); + if (apiUsageState.isReExecEnabled()) { + if (tbMsg.getRuleChainId() == null) { + if (getRootChainActor() != null) { + getRootChainActor().tell(msg); + } else { + tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); + log.info("[{}] No Root Chain: {}", tenantId, msg); + } } else { - tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); - log.info("[{}] No Root Chain: {}", tenantId, msg); + try { + ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg); + } catch (TbActorNotRegisteredException ex) { + log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId()); + //TODO: 3.1 Log it to dead letters queue; + tbMsg.getCallback().onSuccess(); + } } } else { - try { - ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg); - } catch (TbActorNotRegisteredException ex) { - log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId()); - //TODO: 3.1 Log it to dead letters queue; - tbMsg.getCallback().onSuccess(); - } + log.trace("[{}] Ack message because Rule Engine is disabled", tenantId); + tbMsg.getCallback().onSuccess(); } } private void onRuleChainMsg(RuleChainAwareMsg msg) { - getOrCreateActor(msg.getRuleChainId()).tell(msg); + if (apiUsageState.isReExecEnabled()) { + getOrCreateActor(msg.getRuleChainId()).tell(msg); + } } private void onToDeviceActorMsg(DeviceAwareMsg msg, boolean priority) { @@ -215,7 +225,17 @@ public class TenantActor extends RuleChainManagerActor { } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { - if (msg.getEntityId().getEntityType() == EntityType.EDGE) { + if (msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) { + ApiUsageState old = apiUsageState; + apiUsageState = new ApiUsageState(systemContext.getApiUsageStateService().getApiUsageState(tenantId)); + if (old.isReExecEnabled() && !apiUsageState.isReExecEnabled()) { + log.info("[{}] Received API state update. Going to DISABLE Rule Engine execution.", tenantId); + destroyRuleChains(); + } else if (!old.isReExecEnabled() && apiUsageState.isReExecEnabled()) { + log.info("[{}] Received API state update. Going to ENABLE Rule Engine execution.", tenantId); + initRuleChains(); + } + } else if (msg.getEntityId().getEntityType() == EntityType.EDGE) { EdgeId edgeId = new EdgeId(msg.getEntityId().getId()); EdgeRpcService edgeRpcService = systemContext.getEdgeRpcService(); if (msg.getEvent() == ComponentLifecycleEvent.DELETED) { diff --git a/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java b/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java new file mode 100644 index 0000000000..de34db0eaf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java @@ -0,0 +1,267 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.crypto.keygen.Base64StringKeyGenerator; +import org.springframework.security.crypto.keygen.StringKeyGenerator; +import org.springframework.security.oauth2.client.registration.ClientRegistration; +import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository; +import org.springframework.security.oauth2.client.web.OAuth2AuthorizationRequestResolver; +import org.springframework.security.oauth2.core.AuthorizationGrantType; +import org.springframework.security.oauth2.core.ClientAuthenticationMethod; +import org.springframework.security.oauth2.core.endpoint.OAuth2AuthorizationRequest; +import org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames; +import org.springframework.security.oauth2.core.endpoint.PkceParameterNames; +import org.springframework.security.oauth2.core.oidc.OidcScopes; +import org.springframework.security.oauth2.core.oidc.endpoint.OidcParameterNames; +import org.springframework.security.web.util.UrlUtils; +import org.springframework.security.web.util.matcher.AntPathRequestMatcher; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; +import org.thingsboard.server.dao.oauth2.OAuth2Configuration; +import org.thingsboard.server.utils.MiscUtils; + +import javax.servlet.http.HttpServletRequest; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +@Service +@Slf4j +public class CustomOAuth2AuthorizationRequestResolver implements OAuth2AuthorizationRequestResolver { + public static final String DEFAULT_AUTHORIZATION_REQUEST_BASE_URI = "/oauth2/authorization"; + public static final String DEFAULT_LOGIN_PROCESSING_URI = "/login/oauth2/code/"; + private static final String REGISTRATION_ID_URI_VARIABLE_NAME = "registrationId"; + private static final char PATH_DELIMITER = '/'; + + private final AntPathRequestMatcher authorizationRequestMatcher = new AntPathRequestMatcher( + DEFAULT_AUTHORIZATION_REQUEST_BASE_URI + "/{" + REGISTRATION_ID_URI_VARIABLE_NAME + "}"); + private final StringKeyGenerator stateGenerator = new Base64StringKeyGenerator(Base64.getUrlEncoder()); + private final StringKeyGenerator secureKeyGenerator = new Base64StringKeyGenerator(Base64.getUrlEncoder().withoutPadding(), 96); + + @Autowired + private ClientRegistrationRepository clientRegistrationRepository; + + @Autowired(required = false) + private OAuth2Configuration oauth2Configuration; + + + @Override + public OAuth2AuthorizationRequest resolve(HttpServletRequest request) { + String registrationId = this.resolveRegistrationId(request); + String redirectUriAction = getAction(request, "login"); + return resolve(request, registrationId, redirectUriAction); + } + + @Override + public OAuth2AuthorizationRequest resolve(HttpServletRequest request, String registrationId) { + if (registrationId == null) { + return null; + } + String redirectUriAction = getAction(request, "authorize"); + return resolve(request, registrationId, redirectUriAction); + } + + private String getAction(HttpServletRequest request, String defaultAction) { + String action = request.getParameter("action"); + if (action == null) { + return defaultAction; + } + return action; + } + + private OAuth2AuthorizationRequest resolve(HttpServletRequest request, String registrationId, String redirectUriAction) { + if (registrationId == null) { + return null; + } + + ClientRegistration clientRegistration = this.clientRegistrationRepository.findByRegistrationId(registrationId); + if (clientRegistration == null) { + throw new IllegalArgumentException("Invalid Client Registration with Id: " + registrationId); + } + + Map attributes = new HashMap<>(); + attributes.put(OAuth2ParameterNames.REGISTRATION_ID, clientRegistration.getRegistrationId()); + + OAuth2AuthorizationRequest.Builder builder; + if (AuthorizationGrantType.AUTHORIZATION_CODE.equals(clientRegistration.getAuthorizationGrantType())) { + builder = OAuth2AuthorizationRequest.authorizationCode(); + Map additionalParameters = new HashMap<>(); + if (!CollectionUtils.isEmpty(clientRegistration.getScopes()) && + clientRegistration.getScopes().contains(OidcScopes.OPENID)) { + // Section 3.1.2.1 Authentication Request - https://openid.net/specs/openid-connect-core-1_0.html#AuthRequest + // scope + // REQUIRED. OpenID Connect requests MUST contain the "openid" scope value. + addNonceParameters(attributes, additionalParameters); + } + if (ClientAuthenticationMethod.NONE.equals(clientRegistration.getClientAuthenticationMethod())) { + addPkceParameters(attributes, additionalParameters); + } + builder.additionalParameters(additionalParameters); + } else if (AuthorizationGrantType.IMPLICIT.equals(clientRegistration.getAuthorizationGrantType())) { + builder = OAuth2AuthorizationRequest.implicit(); + } else { + throw new IllegalArgumentException("Invalid Authorization Grant Type (" + + clientRegistration.getAuthorizationGrantType().getValue() + + ") for Client Registration with Id: " + clientRegistration.getRegistrationId()); + } + + String redirectUriStr = expandRedirectUri(request, clientRegistration, redirectUriAction); + + return builder + .clientId(clientRegistration.getClientId()) + .authorizationUri(clientRegistration.getProviderDetails().getAuthorizationUri()) + .redirectUri(redirectUriStr) + .scopes(clientRegistration.getScopes()) + .state(this.stateGenerator.generateKey()) + .attributes(attributes) + .build(); + } + + private String resolveRegistrationId(HttpServletRequest request) { + if (this.authorizationRequestMatcher.matches(request)) { + return this.authorizationRequestMatcher + .matcher(request).getVariables().get(REGISTRATION_ID_URI_VARIABLE_NAME); + } + return null; + } + + /** + * Expands the {@link ClientRegistration#getRedirectUriTemplate()} with following provided variables: + * - baseUrl (e.g. https://localhost/app) + * - baseScheme (e.g. https) + * - baseHost (e.g. localhost) + * - basePort (e.g. :8080) + * - basePath (e.g. /app) + * - registrationId (e.g. google) + * - action (e.g. login) + * + * Null variables are provided as empty strings. + * + * Default redirectUriTemplate is: {@link org.springframework.security.config.oauth2.client}.CommonOAuth2Provider#DEFAULT_REDIRECT_URL + * + * @return expanded URI + */ + private String expandRedirectUri(HttpServletRequest request, ClientRegistration clientRegistration, String action) { + Map uriVariables = new HashMap<>(); + uriVariables.put("registrationId", clientRegistration.getRegistrationId()); + + UriComponents uriComponents = UriComponentsBuilder.fromHttpUrl(UrlUtils.buildFullRequestUrl(request)) + .replacePath(request.getContextPath()) + .replaceQuery(null) + .fragment(null) + .build(); + String scheme = uriComponents.getScheme(); + uriVariables.put("baseScheme", scheme == null ? "" : scheme); + String host = uriComponents.getHost(); + uriVariables.put("baseHost", host == null ? "" : host); + // following logic is based on HierarchicalUriComponents#toUriString() + int port = uriComponents.getPort(); + uriVariables.put("basePort", port == -1 ? "" : ":" + port); + String path = uriComponents.getPath(); + if (StringUtils.hasLength(path)) { + if (path.charAt(0) != PATH_DELIMITER) { + path = PATH_DELIMITER + path; + } + } + uriVariables.put("basePath", path == null ? "" : path); + uriVariables.put("baseUrl", uriComponents.toUriString()); + + uriVariables.put("action", action == null ? "" : action); + + String redirectUri = getRedirectUri(request); + log.trace("Redirect URI - {}.", redirectUri); + + return UriComponentsBuilder.fromUriString(redirectUri) + .buildAndExpand(uriVariables) + .toUriString(); + } + + private String getRedirectUri(HttpServletRequest request) { + String loginProcessingUri = oauth2Configuration != null ? oauth2Configuration.getLoginProcessingUrl() : DEFAULT_LOGIN_PROCESSING_URI; + + String scheme = MiscUtils.getScheme(request); + String domainName = MiscUtils.getDomainName(request); + int port = MiscUtils.getPort(request); + String baseUrl = scheme + "://" + domainName; + if (needsPort(scheme, port)){ + baseUrl += ":" + port; + } + return baseUrl + loginProcessingUri; + } + + private boolean needsPort(String scheme, int port) { + boolean isHttpDefault = "http".equals(scheme.toLowerCase()) && port == 80; + boolean isHttpsDefault = "https".equals(scheme.toLowerCase()) && port == 443; + return !isHttpDefault && !isHttpsDefault; + } + + /** + * Creates nonce and its hash for use in OpenID Connect 1.0 Authentication Requests. + * + * @param attributes where the {@link OidcParameterNames#NONCE} is stored for the authentication request + * @param additionalParameters where the {@link OidcParameterNames#NONCE} hash is added for the authentication request + * + * @since 5.2 + * @see 3.1.2.1. Authentication Request + */ + private void addNonceParameters(Map attributes, Map additionalParameters) { + try { + String nonce = this.secureKeyGenerator.generateKey(); + String nonceHash = createHash(nonce); + attributes.put(OidcParameterNames.NONCE, nonce); + additionalParameters.put(OidcParameterNames.NONCE, nonceHash); + } catch (NoSuchAlgorithmException e) { } + } + + /** + * Creates and adds additional PKCE parameters for use in the OAuth 2.0 Authorization and Access Token Requests + * + * @param attributes where {@link PkceParameterNames#CODE_VERIFIER} is stored for the token request + * @param additionalParameters where {@link PkceParameterNames#CODE_CHALLENGE} and, usually, + * {@link PkceParameterNames#CODE_CHALLENGE_METHOD} are added to be used in the authorization request. + * + * @since 5.2 + * @see 1.1. Protocol Flow + * @see 4.1. Client Creates a Code Verifier + * @see 4.2. Client Creates the Code Challenge + */ + private void addPkceParameters(Map attributes, Map additionalParameters) { + String codeVerifier = this.secureKeyGenerator.generateKey(); + attributes.put(PkceParameterNames.CODE_VERIFIER, codeVerifier); + try { + String codeChallenge = createHash(codeVerifier); + additionalParameters.put(PkceParameterNames.CODE_CHALLENGE, codeChallenge); + additionalParameters.put(PkceParameterNames.CODE_CHALLENGE_METHOD, "S256"); + } catch (NoSuchAlgorithmException e) { + additionalParameters.put(PkceParameterNames.CODE_CHALLENGE, codeVerifier); + } + } + + private static String createHash(String value) throws NoSuchAlgorithmException { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] digest = md.digest(value.getBytes(StandardCharsets.US_ASCII)); + return Base64.getUrlEncoder().withoutPadding().encodeToString(digest); + } +} diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 113861e31d..f52a187068 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -32,6 +32,7 @@ import org.springframework.security.config.annotation.web.configuration.EnableWe import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; import org.springframework.security.config.http.SessionCreationPolicy; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; +import org.springframework.security.oauth2.client.web.OAuth2AuthorizationRequestResolver; import org.springframework.security.web.authentication.AuthenticationFailureHandler; import org.springframework.security.web.authentication.AuthenticationSuccessHandler; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; @@ -175,6 +176,9 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt web.ignoring().antMatchers("/*.js","/*.css","/*.ico","/assets/**","/static/**"); } + @Autowired + private OAuth2AuthorizationRequestResolver oAuth2AuthorizationRequestResolver; + @Override protected void configure(HttpSecurity http) throws Exception { http.headers().cacheControl().and().frameOptions().disable() @@ -207,8 +211,10 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt .addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildWsJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class); - if (oauth2Configuration != null && oauth2Configuration.isEnabled()) { + if (oauth2Configuration != null) { http.oauth2Login() + .authorizationEndpoint().authorizationRequestResolver(oAuth2AuthorizationRequestResolver) + .and() .loginPage("/oauth2Login") .loginProcessingUrl(oauth2Configuration.getLoginProcessingUrl()) .successHandler(oauth2AuthenticationSuccessHandler) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 134171c8b1..52283029fe 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -25,6 +25,8 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.SmsService; +import org.thingsboard.server.common.data.sms.config.TestSmsRequest; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -45,6 +47,9 @@ public class AdminController extends BaseController { @Autowired private MailService mailService; + @Autowired + private SmsService smsService; + @Autowired private AdminSettingsService adminSettingsService; @@ -80,6 +85,8 @@ public class AdminController extends BaseController { if (adminSettings.getKey().equals("mail")) { mailService.updateMailConfiguration(); ((ObjectNode) adminSettings.getJsonValue()).put("password", ""); + } else if (adminSettings.getKey().equals("sms")) { + smsService.updateSmsConfiguration(); } return adminSettings; } catch (Exception e) { @@ -127,6 +134,17 @@ public class AdminController extends BaseController { } } + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @RequestMapping(value = "/settings/testSms", method = RequestMethod.POST) + public void sendTestSms(@RequestBody TestSmsRequest testSmsRequest) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + smsService.sendTestSms(testSmsRequest); + } catch (Exception e) { + throw handleException(e); + } + } + @PreAuthorize("hasAuthority('SYS_ADMIN')") @RequestMapping(value = "/updates", method = RequestMethod.GET) @ResponseBody diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index d3d079900a..4f79b85f93 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -133,6 +133,7 @@ public class AlarmController extends BaseController { long ackTs = System.currentTimeMillis(); alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, ackTs).get(); alarm.setAckTs(ackTs); + alarm.setStatus(alarm.getStatus().isCleared() ? AlarmStatus.CLEARED_ACK : AlarmStatus.ACTIVE_ACK); logEntityAction(alarm.getOriginator(), alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_ACK, null); sendNotificationMsgToEdgeService(getTenantId(), alarmId, ActionType.ALARM_ACK); @@ -152,6 +153,7 @@ public class AlarmController extends BaseController { long clearTs = System.currentTimeMillis(); alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, clearTs).get(); alarm.setClearTs(clearTs); + alarm.setStatus(alarm.getStatus().isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK); logEntityAction(alarm.getOriginator(), alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_CLEAR, null); sendNotificationMsgToEdgeService(getTenantId(), alarmId, ActionType.ALARM_CLEAR); diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 24b52b1ec6..10b509adc6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; @@ -55,6 +56,8 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE; + import static org.thingsboard.server.controller.EdgeController.EDGE_ID; @RestController @@ -95,6 +98,10 @@ public class AssetController extends BaseController { @ResponseBody public Asset saveAsset(@RequestBody Asset asset) throws ThingsboardException { try { + if (TB_SERVICE_QUEUE.equals(asset.getType())) { + throw new ThingsboardException("Unable to save asset with type " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } + asset.setTenantId(getCurrentUser().getTenantId()); checkEntity(asset.getId(), asset, Resource.ASSET); diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 4d586c06ad..babce683a2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -39,10 +39,8 @@ import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; -import org.thingsboard.server.dao.oauth2.OAuth2Service; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; @@ -85,9 +83,6 @@ public class AuthController extends BaseController { @Autowired private AuditLogService auditLogService; - @Autowired - private OAuth2Service oauth2Service; - @PreAuthorize("isAuthenticated()") @RequestMapping(value = "/auth/user", method = RequestMethod.GET) public @ResponseBody User getUser() throws ThingsboardException { @@ -175,7 +170,8 @@ public class AuthController extends BaseController { try { String email = resetPasswordByEmailRequest.get("email").asText(); UserCredentials userCredentials = userService.requestPasswordReset(TenantId.SYS_TENANT_ID, email); - String baseUrl = MiscUtils.constructBaseUrl(request); + User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId()); + String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request); String resetUrl = String.format("%s/api/noauth/resetPassword?resetToken=%s", baseUrl, userCredentials.getResetToken()); @@ -223,7 +219,7 @@ public class AuthController extends BaseController { User user = userService.findUserById(TenantId.SYS_TENANT_ID, credentials.getUserId()); UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail()); SecurityUser securityUser = new SecurityUser(user, credentials.isEnabled(), principal); - String baseUrl = MiscUtils.constructBaseUrl(request); + String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request); String loginUrl = String.format("%s/login", baseUrl); String email = user.getEmail(); @@ -272,7 +268,7 @@ public class AuthController extends BaseController { User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId()); UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail()); SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), principal); - String baseUrl = MiscUtils.constructBaseUrl(request); + String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request); String loginUrl = String.format("%s/login", baseUrl); String email = user.getEmail(); mailService.sendPasswordWasResetEmail(loginUrl, email); @@ -342,14 +338,4 @@ public class AuthController extends BaseController { throw handleException(e); } } - - @RequestMapping(value = "/noauth/oauth2Clients", method = RequestMethod.POST) - @ResponseBody - public List getOAuth2Clients() throws ThingsboardException { - try { - return oauth2Service.getOAuth2Clients(); - } catch (Exception e) { - throw handleException(e); - } - } } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index a44d7c0f51..284b88f8cb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -100,6 +100,8 @@ import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService; +import org.thingsboard.server.dao.oauth2.OAuth2Service; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantProfileService; @@ -117,6 +119,7 @@ import org.thingsboard.server.service.edge.EdgeNotificationService; import org.thingsboard.server.service.edge.rpc.EdgeGrpcService; import org.thingsboard.server.service.edge.rpc.init.SyncEdgeService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -186,6 +189,12 @@ public abstract class BaseController { @Autowired protected DashboardService dashboardService; + @Autowired + protected OAuth2Service oAuth2Service; + + @Autowired + protected OAuth2ConfigTemplateService oAuth2ConfigTemplateService; + @Autowired protected ComponentDiscoveryService componentDescriptorService; @@ -222,6 +231,9 @@ public abstract class BaseController { @Autowired protected TbQueueProducerProvider producerProvider; + @Autowired + protected TbTenantProfileCache tenantProfileCache; + @Autowired protected TbDeviceProfileCache deviceProfileCache; @@ -747,6 +759,12 @@ public abstract class BaseController { case ASSIGNED_TO_TENANT: msgType = DataConstants.ENTITY_ASSIGNED_TO_TENANT; break; + case PROVISION_SUCCESS: + msgType = DataConstants.PROVISION_SUCCESS; + break; + case PROVISION_FAILURE: + msgType = DataConstants.PROVISION_FAILURE; + break; case ASSIGNED_TO_EDGE: msgType = DataConstants.ENTITY_ASSIGNED_TO_EDGE; break; diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index e9b41adf06..d1e3a0a478 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -54,6 +54,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; @@ -122,8 +123,11 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); + tbClusterService.onDeviceChange(savedDevice, null); tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); + tbClusterService.onEntityStateChange(savedDevice.getTenantId(), savedDevice.getId(), + device.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); if (device.getId() != null) { sendNotificationMsgToEdgeService(savedDevice.getTenantId(), savedDevice.getId(), ActionType.UPDATED); @@ -156,6 +160,9 @@ public class DeviceController extends BaseController { Device device = checkDeviceId(deviceId, Operation.DELETE); deviceService.deleteDevice(getCurrentUser().getTenantId(), deviceId); + tbClusterService.onDeviceDeleted(device, null); + tbClusterService.onEntityStateChange(device.getTenantId(), deviceId, ComponentLifecycleEvent.DELETED); + logEntityAction(deviceId, device, device.getCustomerId(), ActionType.DELETED, null, strDeviceId); diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java index b474a0c0f1..fcb34530b5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java @@ -94,7 +94,6 @@ public class DeviceProfileController extends BaseController { DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile)); - deviceProfileCache.put(savedDeviceProfile); tbClusterService.onDeviceProfileChange(savedDeviceProfile, null); tbClusterService.onEntityStateChange(deviceProfile.getTenantId(), savedDeviceProfile.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); @@ -120,7 +119,6 @@ public class DeviceProfileController extends BaseController { DeviceProfileId deviceProfileId = new DeviceProfileId(toUUID(strDeviceProfileId)); DeviceProfile deviceProfile = checkDeviceProfileId(deviceProfileId, Operation.DELETE); deviceProfileService.deleteDeviceProfile(getTenantId(), deviceProfileId); - deviceProfileCache.evict(deviceProfileId); tbClusterService.onDeviceProfileDelete(deviceProfile, null); tbClusterService.onEntityStateChange(deviceProfile.getTenantId(), deviceProfile.getId(), ComponentLifecycleEvent.DELETED); @@ -192,10 +190,11 @@ public class DeviceProfileController extends BaseController { @RequestParam int page, @RequestParam(required = false) String textSearch, @RequestParam(required = false) String sortProperty, - @RequestParam(required = false) String sortOrder) throws ThingsboardException { + @RequestParam(required = false) String sortOrder, + @RequestParam(required = false) String transportType) throws ThingsboardException { try { PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); - return checkNotNull(deviceProfileService.findDeviceProfileInfos(getTenantId(), pageLink)); + return checkNotNull(deviceProfileService.findDeviceProfileInfos(getTenantId(), pageLink, transportType)); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/OAuth2ConfigTemplateController.java b/application/src/main/java/org/thingsboard/server/controller/OAuth2ConfigTemplateController.java new file mode 100644 index 0000000000..35ed01f03e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/OAuth2ConfigTemplateController.java @@ -0,0 +1,77 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.*; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.OAuth2ClientRegistrationTemplateId; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationTemplate; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; + +import java.util.List; + +@RestController +@TbCoreComponent +@RequestMapping("/api/oauth2/config/template") +@Slf4j +public class OAuth2ConfigTemplateController extends BaseController { + private static final String CLIENT_REGISTRATION_TEMPLATE_ID = "clientRegistrationTemplateId"; + + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @RequestMapping(method = RequestMethod.POST) + @ResponseStatus(value = HttpStatus.OK) + public OAuth2ClientRegistrationTemplate saveClientRegistrationTemplate(@RequestBody OAuth2ClientRegistrationTemplate clientRegistrationTemplate) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_TEMPLATE, Operation.WRITE); + return oAuth2ConfigTemplateService.saveClientRegistrationTemplate(clientRegistrationTemplate); + } catch (Exception e) { + throw handleException(e); + } + } + + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @RequestMapping(value = "/{clientRegistrationTemplateId}", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteClientRegistrationTemplate(@PathVariable(CLIENT_REGISTRATION_TEMPLATE_ID) String strClientRegistrationTemplateId) throws ThingsboardException { + checkParameter(CLIENT_REGISTRATION_TEMPLATE_ID, strClientRegistrationTemplateId); + try { + accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_TEMPLATE, Operation.DELETE); + OAuth2ClientRegistrationTemplateId clientRegistrationTemplateId = new OAuth2ClientRegistrationTemplateId(toUUID(strClientRegistrationTemplateId)); + oAuth2ConfigTemplateService.deleteClientRegistrationTemplateById(clientRegistrationTemplateId); + } catch (Exception e) { + throw handleException(e); + } + } + + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") + @RequestMapping(method = RequestMethod.GET, produces = "application/json") + @ResponseBody + public List getClientRegistrationTemplates() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_TEMPLATE, Operation.READ); + return oAuth2ConfigTemplateService.findAllClientRegistrationTemplates(); + } catch (Exception e) { + throw handleException(e); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/controller/OAuth2Controller.java b/application/src/main/java/org/thingsboard/server/controller/OAuth2Controller.java new file mode 100644 index 0000000000..fc5e0a1426 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/OAuth2Controller.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.*; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientsParams; +import org.thingsboard.server.common.data.oauth2.SchemeType; +import org.thingsboard.server.dao.oauth2.OAuth2Configuration; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; +import org.thingsboard.server.utils.MiscUtils; + +import javax.servlet.http.HttpServletRequest; +import java.util.List; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +@Slf4j +public class OAuth2Controller extends BaseController { + + @Autowired + private OAuth2Configuration oAuth2Configuration; + + @RequestMapping(value = "/noauth/oauth2Clients", method = RequestMethod.POST) + @ResponseBody + public List getOAuth2Clients(HttpServletRequest request) throws ThingsboardException { + try { + return oAuth2Service.getOAuth2Clients(MiscUtils.getScheme(request), MiscUtils.getDomainNameAndPort(request)); + } catch (Exception e) { + throw handleException(e); + } + } + + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @RequestMapping(value = "/oauth2/config", method = RequestMethod.GET, produces = "application/json") + @ResponseBody + public OAuth2ClientsParams getCurrentOAuth2Params() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_INFO, Operation.READ); + return oAuth2Service.findOAuth2Params(); + } catch (Exception e) { + throw handleException(e); + } + } + + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @RequestMapping(value = "/oauth2/config", method = RequestMethod.POST) + @ResponseStatus(value = HttpStatus.OK) + public OAuth2ClientsParams saveOAuth2Params(@RequestBody OAuth2ClientsParams oauth2Params) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_INFO, Operation.WRITE); + oAuth2Service.saveOAuth2Params(oauth2Params); + return oAuth2Service.findOAuth2Params(); + } catch (Exception e) { + throw handleException(e); + } + } + + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @RequestMapping(value = "/oauth2/loginProcessingUrl", method = RequestMethod.GET) + @ResponseBody + public String getLoginProcessingUrl() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_INFO, Operation.READ); + return "\"" + oAuth2Configuration.getLoginProcessingUrl() + "\""; + } catch (Exception e) { + throw handleException(e); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 20b3fa60ea..2beaf0ddd2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -173,10 +173,12 @@ public class RuleChainController extends BaseController { public RuleChain saveRuleChain(@RequestBody DefaultRuleChainCreateRequest request) throws ThingsboardException { try { checkNotNull(request); - checkNotNull(request.getName()); + checkParameter(request.getName(), "name"); RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName()); + tbClusterService.onEntityStateChange(savedRuleChain.getTenantId(), savedRuleChain.getId(), ComponentLifecycleEvent.CREATED); + logEntityAction(savedRuleChain.getId(), savedRuleChain, null, ActionType.ADDED, null); return savedRuleChain; @@ -377,7 +379,7 @@ public class RuleChainController extends BaseController { String errorText = ""; ScriptEngine engine = null; try { - engine = new RuleNodeJsScriptEngine(jsInvokeService, getCurrentUser().getId(), script, argNames); + engine = new RuleNodeJsScriptEngine(getTenantId(), jsInvokeService, getCurrentUser().getId(), script, argNames); TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); switch (scriptType) { case "update": diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index c8c41ea479..2141027fa7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -45,6 +45,7 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceId; @@ -69,6 +70,7 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -93,6 +95,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -205,7 +208,7 @@ public class TelemetryController extends BaseController { @RequestParam(name = "interval", defaultValue = "0") Long interval, @RequestParam(name = "limit", defaultValue = "100") Integer limit, @RequestParam(name = "agg", defaultValue = "NONE") String aggStr, - @RequestParam(name= "orderBy", defaultValue = "DESC") String orderBy, + @RequestParam(name = "orderBy", defaultValue = "DESC") String orderBy, @RequestParam(name = "useStrictDataTypes", required = false, defaultValue = "false") Boolean useStrictDataTypes) throws ThingsboardException { return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr, (result, tenantId, entityId) -> { @@ -392,6 +395,11 @@ public class TelemetryController extends BaseController { if (attributes.isEmpty()) { return getImmediateDeferredResult("No attributes data found in request body!", HttpStatus.BAD_REQUEST); } + for (AttributeKvEntry attributeKvEntry : attributes) { + if (attributeKvEntry.getKey().isEmpty() || attributeKvEntry.getKey().trim().length() == 0) { + return getImmediateDeferredResult("Key cannot be empty or contains only spaces", HttpStatus.BAD_REQUEST); + } + } SecurityUser user = getCurrentUser(); return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { tsSubService.saveAndNotify(tenantId, entityId, scope, attributes, new FutureCallback() { @@ -435,9 +443,13 @@ public class TelemetryController extends BaseController { if (entries.isEmpty()) { return getImmediateDeferredResult("No timeseries data found in request body!", HttpStatus.BAD_REQUEST); } - SecurityUser user = getCurrentUser(); return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_TELEMETRY, entityIdSrc, (result, tenantId, entityId) -> { - tsSubService.saveAndNotify(tenantId, entityId, entries, ttl, new FutureCallback() { + long tenantTtl = ttl; + if (!TenantId.SYS_TENANT_ID.equals(tenantId) && tenantTtl == 0) { + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + tenantTtl = TimeUnit.DAYS.toSeconds(((DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration()).getDefaultStorageTtlDays()); + } + tsSubService.saveAndNotify(tenantId, entityId, entries, tenantTtl, new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { result.setResult(new ResponseEntity(HttpStatus.OK)); diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index ebb46778c9..a42cbb0f91 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -91,6 +91,10 @@ public class TenantController extends BaseController { if (newTenant) { installScripts.createDefaultRuleChains(tenant.getId()); } + tenantProfileCache.evict(tenant.getId()); + tbClusterService.onTenantChange(tenant, null); + tbClusterService.onEntityStateChange(tenant.getId(), tenant.getId(), + newTenant ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); return tenant; } catch (Exception e) { throw handleException(e); @@ -104,8 +108,10 @@ public class TenantController extends BaseController { checkParameter("tenantId", strTenantId); try { TenantId tenantId = new TenantId(toUUID(strTenantId)); - checkTenantId(tenantId, Operation.DELETE); + Tenant tenant = checkTenantId(tenantId, Operation.DELETE); tenantService.deleteTenant(tenantId); + tenantProfileCache.evict(tenantId); + tbClusterService.onTenantDelete(tenant, null); tbClusterService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java b/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java index dfe0b19a42..0941fac725 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java @@ -29,9 +29,12 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -93,8 +96,12 @@ public class TenantProfileController extends BaseController { } tenantProfile = checkNotNull(tenantProfileService.saveTenantProfile(getTenantId(), tenantProfile)); + tenantProfileCache.put(tenantProfile); + tbClusterService.onTenantProfileChange(tenantProfile, null); + tbClusterService.onEntityStateChange(TenantId.SYS_TENANT_ID, tenantProfile.getId(), + newTenantProfile ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); return tenantProfile; - } catch (Exception e) { + } catch (Exception e) { throw handleException(e); } } @@ -106,8 +113,9 @@ public class TenantProfileController extends BaseController { checkParameter("tenantProfileId", strTenantProfileId); try { TenantProfileId tenantProfileId = new TenantProfileId(toUUID(strTenantProfileId)); - checkTenantProfileId(tenantProfileId, Operation.DELETE); + TenantProfile profile = checkTenantProfileId(tenantProfileId, Operation.DELETE); tenantProfileService.deleteTenantProfile(getTenantId(), tenantProfileId); + tbClusterService.onTenantProfileDelete(profile, null); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index 47c0d735fb..cc47656b6e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -52,6 +52,7 @@ import org.thingsboard.server.service.security.model.token.JwtToken; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.utils.MiscUtils; import javax.servlet.http.HttpServletRequest; @@ -78,6 +79,9 @@ public class UserController extends BaseController { @Autowired private RefreshTokenRepository refreshTokenRepository; + @Autowired + private SystemSecurityService systemSecurityService; + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/user/{userId}", method = RequestMethod.GET) @@ -146,7 +150,7 @@ public class UserController extends BaseController { if (sendEmail) { SecurityUser authUser = getCurrentUser(); UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), savedUser.getId()); - String baseUrl = MiscUtils.constructBaseUrl(request); + String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request); String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, userCredentials.getActivateToken()); String email = savedUser.getEmail(); @@ -189,7 +193,7 @@ public class UserController extends BaseController { UserCredentials userCredentials = userService.findUserCredentialsByUserId(getCurrentUser().getTenantId(), user.getId()); if (!userCredentials.isEnabled()) { - String baseUrl = MiscUtils.constructBaseUrl(request); + String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request); String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, userCredentials.getActivateToken()); mailService.sendActivationEmail(activateUrl, email); @@ -214,7 +218,7 @@ public class UserController extends BaseController { SecurityUser authUser = getCurrentUser(); UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), user.getId()); if (!userCredentials.isEnabled()) { - String baseUrl = MiscUtils.constructBaseUrl(request); + String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request); String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, userCredentials.getActivateToken()); return activateUrl; diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 2b4fae6045..b4d5a0ff94 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -194,6 +194,7 @@ public class ThingsboardInstallService { dataUpdateService.updateData("3.1.1"); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); + systemDataLoaderService.createOAuth2Templates(); break; default: throw new RuntimeException("Unable to upgrade ThingsBoard, unsupported fromVersion: " + upgradeFromVersion); @@ -226,6 +227,7 @@ public class ThingsboardInstallService { systemDataLoaderService.createDefaultTenantProfiles(); systemDataLoaderService.createAdminSettings(); systemDataLoaderService.loadSystemWidgets(); + systemDataLoaderService.createOAuth2Templates(); // systemDataLoaderService.loadSystemPlugins(); // systemDataLoaderService.loadSystemRules(); diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java new file mode 100644 index 0000000000..12e8e6017d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -0,0 +1,450 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import com.google.common.util.concurrent.FutureCallback; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.server.common.data.ApiFeature; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.ApiUsageStateMailMessage; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.tools.SchedulerUtils; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.telemetry.InternalTelemetryService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +@Slf4j +@Service +public class DefaultTbApiUsageStateService implements TbApiUsageStateService { + + public static final String HOURLY = "Hourly"; + public static final FutureCallback VOID_CALLBACK = new FutureCallback() { + @Override + public void onSuccess(@Nullable Integer result) { + } + + @Override + public void onFailure(Throwable t) { + } + }; + private final TbClusterService clusterService; + private final PartitionService partitionService; + private final TenantService tenantService; + private final TimeseriesService tsService; + private final ApiUsageStateService apiUsageStateService; + private final SchedulerComponent scheduler; + private final TbTenantProfileCache tenantProfileCache; + private final MailService mailService; + + @Lazy + @Autowired + private InternalTelemetryService tsWsService; + + // Tenants that should be processed on this server + private final Map myTenantStates = new ConcurrentHashMap<>(); + // Tenants that should be processed on other servers + private final Map otherTenantStates = new ConcurrentHashMap<>(); + + @Value("${usage.stats.report.enabled:true}") + private boolean enabled; + + @Value("${usage.stats.check.cycle:60000}") + private long nextCycleCheckInterval; + + private final Lock updateLock = new ReentrantLock(); + + private final ExecutorService mailExecutor; + + public DefaultTbApiUsageStateService(TbClusterService clusterService, + PartitionService partitionService, + TenantService tenantService, + TimeseriesService tsService, + ApiUsageStateService apiUsageStateService, + SchedulerComponent scheduler, + TbTenantProfileCache tenantProfileCache, MailService mailService) { + this.clusterService = clusterService; + this.partitionService = partitionService; + this.tenantService = tenantService; + this.tsService = tsService; + this.apiUsageStateService = apiUsageStateService; + this.scheduler = scheduler; + this.tenantProfileCache = tenantProfileCache; + this.mailService = mailService; + this.mailExecutor = Executors.newSingleThreadExecutor(); + } + + @PostConstruct + public void init() { + if (enabled) { + log.info("Starting api usage service."); + scheduler.scheduleAtFixedRate(this::checkStartOfNextCycle, nextCycleCheckInterval, nextCycleCheckInterval, TimeUnit.MILLISECONDS); + log.info("Started api usage service."); + } + } + + @Override + public void process(TbProtoQueueMsg msg, TbCallback callback) { + ToUsageStatsServiceMsg statsMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB())); + + if (tenantProfileCache.get(tenantId) == null) { + return; + } + + TenantApiUsageState tenantState; + List updatedEntries; + Map result; + updateLock.lock(); + try { + tenantState = getOrFetchState(tenantId); + long ts = tenantState.getCurrentCycleTs(); + long hourTs = tenantState.getCurrentHourTs(); + long newHourTs = SchedulerUtils.getStartOfCurrentHour(); + if (newHourTs != hourTs) { + tenantState.setHour(newHourTs); + } + updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length); + Set apiFeatures = new HashSet<>(); + for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) { + ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey()); + long newValue = tenantState.add(recordKey, kvProto.getValue()); + updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.getApiCountKey(), newValue))); + long newHourlyValue = tenantState.addToHourly(recordKey, kvProto.getValue()); + updatedEntries.add(new BasicTsKvEntry(newHourTs, new LongDataEntry(recordKey.getApiCountKey() + HOURLY, newHourlyValue))); + apiFeatures.add(recordKey.getApiFeature()); + } + result = tenantState.checkStateUpdatedDueToThreshold(apiFeatures); + } finally { + updateLock.unlock(); + } + tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK); + if (!result.isEmpty()) { + persistAndNotify(tenantState, result); + } + callback.onSuccess(); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) { + myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); + otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); + initStatesFromDataBase(); + } + } + + @Override + public ApiUsageState getApiUsageState(TenantId tenantId) { + TenantApiUsageState tenantState = myTenantStates.get(tenantId); + if (tenantState != null) { + return tenantState.getApiUsageState(); + } else { + ApiUsageState state = otherTenantStates.get(tenantId); + if (state != null) { + return state; + } else { + if (partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { + return getOrFetchState(tenantId).getApiUsageState(); + } else { + updateLock.lock(); + try { + state = otherTenantStates.get(tenantId); + if (state == null) { + state = apiUsageStateService.findTenantApiUsageState(tenantId); + if (state != null) { + otherTenantStates.put(tenantId, state); + } + } + } finally { + updateLock.unlock(); + } + return state; + } + } + } + } + + @Override + public void onApiUsageStateUpdate(TenantId tenantId) { + otherTenantStates.remove(tenantId); + } + + @Override + public void onTenantProfileUpdate(TenantProfileId tenantProfileId) { + log.info("[{}] On Tenant Profile Update", tenantProfileId); + TenantProfile tenantProfile = tenantProfileCache.get(tenantProfileId); + updateLock.lock(); + try { + myTenantStates.values().forEach(state -> { + if (tenantProfile.getId().equals(state.getTenantProfileId())) { + updateTenantState(state, tenantProfile); + } + }); + } finally { + updateLock.unlock(); + } + } + + @Override + public void onTenantUpdate(TenantId tenantId) { + log.info("[{}] On Tenant Update.", tenantId); + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + updateLock.lock(); + try { + TenantApiUsageState state = myTenantStates.get(tenantId); + if (state != null && !state.getTenantProfileId().equals(tenantProfile.getId())) { + updateTenantState(state, tenantProfile); + } + } finally { + updateLock.unlock(); + } + } + + private void updateTenantState(TenantApiUsageState state, TenantProfile profile) { + TenantProfileData oldProfileData = state.getTenantProfileData(); + state.setTenantProfileId(profile.getId()); + state.setTenantProfileData(profile.getProfileData()); + Map result = state.checkStateUpdatedDueToThresholds(); + if (!result.isEmpty()) { + persistAndNotify(state, result); + } + updateProfileThresholds(state.getTenantId(), state.getApiUsageState().getId(), + oldProfileData.getConfiguration(), profile.getProfileData().getConfiguration()); + } + + private void updateProfileThresholds(TenantId tenantId, ApiUsageStateId id, + TenantProfileConfiguration oldData, TenantProfileConfiguration newData) { + long ts = System.currentTimeMillis(); + List profileThresholds = new ArrayList<>(); + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + long newProfileThreshold = newData.getProfileThreshold(key); + if (oldData == null || oldData.getProfileThreshold(key) != newProfileThreshold) { + log.info("[{}] Updating profile threshold [{}]:[{}]", tenantId, key, newProfileThreshold); + profileThresholds.add(new BasicTsKvEntry(ts, new LongDataEntry(key.getApiLimitKey(), newProfileThreshold))); + } + } + if (!profileThresholds.isEmpty()) { + tsWsService.saveAndNotifyInternal(tenantId, id, profileThresholds, VOID_CALLBACK); + } + } + + private void persistAndNotify(TenantApiUsageState state, Map result) { + log.info("[{}] Detected update of the API state: {}", state.getTenantId(), result); + apiUsageStateService.update(state.getApiUsageState()); + clusterService.onApiStateChange(state.getApiUsageState(), null); + long ts = System.currentTimeMillis(); + List stateTelemetry = new ArrayList<>(); + result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(apiFeature.getApiStateKey(), aState.name()))))); + tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, VOID_CALLBACK); + + String email = tenantService.findTenantById(state.getTenantId()).getEmail(); + + if (StringUtils.isNotEmpty(email)) { + result.forEach((apiFeature, stateValue) -> { + mailExecutor.submit(() -> { + try { + mailService.sendApiFeatureStateEmail(apiFeature, stateValue, email, createStateMailMessage(state, apiFeature, stateValue)); + } catch (ThingsboardException e) { + log.warn("[{}] Can't send update of the API state to tenant with provided email [{}]", state.getTenantId(), email, e); + } + }); + }); + } else { + log.warn("[{}] Can't send update of the API state to tenant with empty email!", state.getTenantId()); + } + } + + private ApiUsageStateMailMessage createStateMailMessage(TenantApiUsageState state, ApiFeature apiFeature, ApiUsageStateValue stateValue) { + StateChecker checker = getStateChecker(stateValue); + for (ApiUsageRecordKey apiUsageRecordKey : ApiUsageRecordKey.getKeys(apiFeature)) { + long threshold = state.getProfileThreshold(apiUsageRecordKey); + long warnThreshold = state.getProfileWarnThreshold(apiUsageRecordKey); + long value = state.get(apiUsageRecordKey); + if (checker.check(threshold, warnThreshold, value)) { + return new ApiUsageStateMailMessage(apiUsageRecordKey, threshold, value); + } + } + return null; + } + + private StateChecker getStateChecker(ApiUsageStateValue stateValue) { + if (ApiUsageStateValue.ENABLED.equals(stateValue)) { + return (t, wt, v) -> true; + } else if (ApiUsageStateValue.WARNING.equals(stateValue)) { + return (t, wt, v) -> v < t && v >= wt; + } else { + return (t, wt, v) -> v >= t; + } + } + + private interface StateChecker { + boolean check(long threshold, long warnThreshold, long value); + } + + private void checkStartOfNextCycle() { + updateLock.lock(); + try { + long now = System.currentTimeMillis(); + myTenantStates.values().forEach(state -> { + if ((state.getNextCycleTs() < now) && (now - state.getNextCycleTs() < TimeUnit.HOURS.toMillis(1))) { + TenantId tenantId = state.getTenantId(); + state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextNextMonth()); + saveNewCounts(state, Arrays.asList(ApiUsageRecordKey.values())); + updateTenantState(state, tenantProfileCache.get(tenantId)); + } + }); + } finally { + updateLock.unlock(); + } + } + + private void saveNewCounts(TenantApiUsageState state, List keys) { + List counts = keys.stream() + .map(key -> new BasicTsKvEntry(state.getCurrentCycleTs(), new LongDataEntry(key.getApiCountKey(), 0L))) + .collect(Collectors.toList()); + + tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), counts, VOID_CALLBACK); + } + + private TenantApiUsageState getOrFetchState(TenantId tenantId) { + TenantApiUsageState tenantState = myTenantStates.get(tenantId); + if (tenantState == null) { + ApiUsageState dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); + if (dbStateEntity == null) { + try { + dbStateEntity = apiUsageStateService.createDefaultApiUsageState(tenantId); + } catch (Exception e) { + dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); + } + } + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + tenantState = new TenantApiUsageState(tenantProfile, dbStateEntity); + List newCounts = new ArrayList<>(); + try { + List dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getId()).get(); + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + boolean cycleEntryFound = false; + boolean hourlyEntryFound = false; + for (TsKvEntry tsKvEntry : dbValues) { + if (tsKvEntry.getKey().equals(key.getApiCountKey())) { + cycleEntryFound = true; + + boolean oldCount = tsKvEntry.getTs() == tenantState.getCurrentCycleTs(); + tenantState.put(key, oldCount ? tsKvEntry.getLongValue().get() : 0L); + + if (!oldCount) { + newCounts.add(key); + } + } else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) { + hourlyEntryFound = true; + tenantState.putHourly(key, tsKvEntry.getTs() == tenantState.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L); + } + if (cycleEntryFound && hourlyEntryFound) { + break; + } + } + } + log.debug("[{}] Initialized state: {}", tenantId, dbStateEntity); + myTenantStates.put(tenantId, tenantState); + saveNewCounts(tenantState, newCounts); + } catch (InterruptedException | ExecutionException e) { + log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e); + } + } + return tenantState; + } + + private void initStatesFromDataBase() { + try { + log.info("Initializing tenant states."); + PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, 1024); + for (Tenant tenant : tenantIterator) { + if (!myTenantStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) { + log.debug("[{}] Initializing tenant state.", tenant.getId()); + updateLock.lock(); + try { + updateTenantState(getOrFetchState(tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId())); + log.debug("[{}] Initialized tenant state.", tenant.getId()); + } catch (Exception e) { + log.warn("[{}] Failed to initialize tenant API state", tenant.getId(), e); + } finally { + updateLock.unlock(); + } + } + } + log.info("Initialized tenant states."); + } catch (Exception e) { + log.warn("Unknown failure", e); + } + } + + @PreDestroy + private void destroy() { + if (mailExecutor != null) { + mailExecutor.shutdownNow(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java new file mode 100644 index 0000000000..e19c0a4154 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; + +public interface TbApiUsageStateService extends ApplicationListener { + + void process(TbProtoQueueMsg msg, TbCallback callback); + + ApiUsageState getApiUsageState(TenantId tenantId); + + void onTenantProfileUpdate(TenantProfileId tenantProfileId); + + void onTenantUpdate(TenantId tenantId); + + void onApiUsageStateUpdate(TenantId tenantId); +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java new file mode 100644 index 0000000000..fd4ef96122 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java @@ -0,0 +1,196 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.data.util.Pair; +import org.thingsboard.server.common.data.ApiFeature; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.msg.tools.SchedulerUtils; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class TenantApiUsageState { + + private final Map currentCycleValues = new ConcurrentHashMap<>(); + private final Map currentHourValues = new ConcurrentHashMap<>(); + + @Getter + @Setter + private TenantProfileId tenantProfileId; + @Getter + @Setter + private TenantProfileData tenantProfileData; + @Getter + private final ApiUsageState apiUsageState; + @Getter + private volatile long currentCycleTs; + @Getter + private volatile long nextCycleTs; + @Getter + private volatile long currentHourTs; + + public TenantApiUsageState(TenantProfile tenantProfile, ApiUsageState apiUsageState) { + this.tenantProfileId = tenantProfile.getId(); + this.tenantProfileData = tenantProfile.getProfileData(); + this.apiUsageState = apiUsageState; + this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth(); + this.nextCycleTs = SchedulerUtils.getStartOfNextMonth(); + this.currentHourTs = SchedulerUtils.getStartOfCurrentHour(); + } + + public void put(ApiUsageRecordKey key, Long value) { + currentCycleValues.put(key, value); + } + + public void putHourly(ApiUsageRecordKey key, Long value) { + currentHourValues.put(key, value); + } + + public long add(ApiUsageRecordKey key, long value) { + long result = currentCycleValues.getOrDefault(key, 0L) + value; + currentCycleValues.put(key, result); + return result; + } + + public long get(ApiUsageRecordKey key) { + return currentCycleValues.getOrDefault(key, 0L); + } + + public long addToHourly(ApiUsageRecordKey key, long value) { + long result = currentHourValues.getOrDefault(key, 0L) + value; + currentHourValues.put(key, result); + return result; + } + + public void setHour(long currentHourTs) { + this.currentHourTs = currentHourTs; + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + currentHourValues.put(key, 0L); + } + } + + public void setCycles(long currentCycleTs, long nextCycleTs) { + this.currentCycleTs = currentCycleTs; + this.nextCycleTs = nextCycleTs; + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + currentCycleValues.put(key, 0L); + } + } + + public long getProfileThreshold(ApiUsageRecordKey key) { + return tenantProfileData.getConfiguration().getProfileThreshold(key); + } + + public long getProfileWarnThreshold(ApiUsageRecordKey key) { + return tenantProfileData.getConfiguration().getWarnThreshold(key); + } + + public TenantId getTenantId() { + return apiUsageState.getTenantId(); + } + + public ApiUsageStateValue getFeatureValue(ApiFeature feature) { + switch (feature) { + case TRANSPORT: + return apiUsageState.getTransportState(); + case RE: + return apiUsageState.getReExecState(); + case DB: + return apiUsageState.getDbStorageState(); + case JS: + return apiUsageState.getJsExecState(); + case EMAIL: + return apiUsageState.getEmailExecState(); + case SMS: + return apiUsageState.getSmsExecState(); + default: + return ApiUsageStateValue.ENABLED; + } + } + + public boolean setFeatureValue(ApiFeature feature, ApiUsageStateValue value) { + ApiUsageStateValue currentValue = getFeatureValue(feature); + switch (feature) { + case TRANSPORT: + apiUsageState.setTransportState(value); + break; + case RE: + apiUsageState.setReExecState(value); + break; + case DB: + apiUsageState.setDbStorageState(value); + break; + case JS: + apiUsageState.setJsExecState(value); + break; + case EMAIL: + apiUsageState.setEmailExecState(value); + break; + case SMS: + apiUsageState.setSmsExecState(value); + break; + } + return !currentValue.equals(value); + } + + public Map checkStateUpdatedDueToThresholds() { + return checkStateUpdatedDueToThreshold(new HashSet<>(Arrays.asList(ApiFeature.values()))); + } + + public Map checkStateUpdatedDueToThreshold(Set features) { + Map result = new HashMap<>(); + for (ApiFeature feature : features) { + Pair tmp = checkStateUpdatedDueToThreshold(feature); + if (tmp != null) { + result.put(tmp.getFirst(), tmp.getSecond()); + } + } + return result; + } + + public Pair checkStateUpdatedDueToThreshold(ApiFeature feature) { + ApiUsageStateValue featureValue = ApiUsageStateValue.ENABLED; + for (ApiUsageRecordKey recordKey : ApiUsageRecordKey.getKeys(feature)) { + long value = get(recordKey); + long threshold = getProfileThreshold(recordKey); + long warnThreshold = getProfileWarnThreshold(recordKey); + ApiUsageStateValue tmpValue; + if (threshold == 0 || value < warnThreshold) { + tmpValue = ApiUsageStateValue.ENABLED; + } else if (value < threshold) { + tmpValue = ApiUsageStateValue.WARNING; + } else { + tmpValue = ApiUsageStateValue.DISABLED; + } + featureValue = ApiUsageStateValue.toMoreRestricted(featureValue, tmpValue); + } + return setFeatureValue(feature, featureValue) ? Pair.of(feature, featureValue) : null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java new file mode 100644 index 0000000000..2a2f1bd03a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java @@ -0,0 +1,268 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.device; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.RandomStringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.device.DeviceDao; +import org.thingsboard.server.dao.device.DeviceProfileDao; +import org.thingsboard.server.dao.device.DeviceProvisionService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.device.provision.ProvisionFailedException; +import org.thingsboard.server.dao.device.provision.ProvisionRequest; +import org.thingsboard.server.dao.device.provision.ProvisionResponse; +import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus; +import org.thingsboard.server.dao.util.mapping.JacksonUtil; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.state.DeviceStateService; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.locks.ReentrantLock; + + +@Service +@Slf4j +@TbCoreComponent +public class DeviceProvisionServiceImpl implements DeviceProvisionService { + + protected TbQueueProducer> ruleEngineMsgProducer; + + private static final String DEVICE_PROVISION_STATE = "provisionState"; + private static final String PROVISIONED_STATE = "provisioned"; + + private final ReentrantLock deviceCreationLock = new ReentrantLock(); + + @Autowired + DeviceDao deviceDao; + + @Autowired + DeviceProfileDao deviceProfileDao; + + @Autowired + DeviceService deviceService; + + @Autowired + DeviceCredentialsService deviceCredentialsService; + + @Autowired + AttributesService attributesService; + + @Autowired + DeviceStateService deviceStateService; + + @Autowired + AuditLogService auditLogService; + + @Autowired + PartitionService partitionService; + + public DeviceProvisionServiceImpl(TbQueueProducerProvider producerProvider) { + ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); + } + + @Override + public ProvisionResponse provisionDevice(ProvisionRequest provisionRequest) { + String provisionRequestKey = provisionRequest.getCredentials().getProvisionDeviceKey(); + String provisionRequestSecret = provisionRequest.getCredentials().getProvisionDeviceSecret(); + if (!StringUtils.isEmpty(provisionRequest.getDeviceName())) { + provisionRequest.setDeviceName(provisionRequest.getDeviceName().trim()); + if (StringUtils.isEmpty(provisionRequest.getDeviceName())) { + log.warn("Provision request contains empty device name!"); + throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + } + } + + if (StringUtils.isEmpty(provisionRequestKey) || StringUtils.isEmpty(provisionRequestSecret)) { + throw new ProvisionFailedException(ProvisionResponseStatus.NOT_FOUND.name()); + } + + DeviceProfile targetProfile = deviceProfileDao.findByProvisionDeviceKey(provisionRequestKey); + + if (targetProfile == null || targetProfile.getProfileData().getProvisionConfiguration() == null || + targetProfile.getProfileData().getProvisionConfiguration().getProvisionDeviceSecret() == null) { + throw new ProvisionFailedException(ProvisionResponseStatus.NOT_FOUND.name()); + } + + Device targetDevice = deviceDao.findDeviceByTenantIdAndName(targetProfile.getTenantId().getId(), provisionRequest.getDeviceName()).orElse(null); + + switch (targetProfile.getProvisionType()) { + case ALLOW_CREATE_NEW_DEVICES: + if (targetProfile.getProfileData().getProvisionConfiguration().getProvisionDeviceSecret().equals(provisionRequestSecret)) { + if (targetDevice != null) { + log.warn("[{}] The device is present and could not be provisioned once more!", targetDevice.getName()); + notify(targetDevice, provisionRequest, DataConstants.PROVISION_FAILURE, false); + throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + } else { + return createDevice(provisionRequest, targetProfile); + } + } + break; + case CHECK_PRE_PROVISIONED_DEVICES: + if (targetProfile.getProfileData().getProvisionConfiguration().getProvisionDeviceSecret().equals(provisionRequestSecret)) { + if (targetDevice != null && targetDevice.getDeviceProfileId().equals(targetProfile.getId())) { + return processProvision(targetDevice, provisionRequest); + } else { + log.warn("[{}] Failed to find pre provisioned device!", provisionRequest.getDeviceName()); + throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + } + } + break; + } + throw new ProvisionFailedException(ProvisionResponseStatus.NOT_FOUND.name()); + } + + private ProvisionResponse processProvision(Device device, ProvisionRequest provisionRequest) { + try { + Optional provisionState = attributesService.find(device.getTenantId(), device.getId(), + DataConstants.SERVER_SCOPE, DEVICE_PROVISION_STATE).get(); + if (provisionState != null && provisionState.isPresent() && !provisionState.get().getValueAsString().equals(PROVISIONED_STATE)) { + notify(device, provisionRequest, DataConstants.PROVISION_FAILURE, false); + throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + } else { + saveProvisionStateAttribute(device).get(); + notify(device, provisionRequest, DataConstants.PROVISION_SUCCESS, true); + } + } catch (InterruptedException | ExecutionException e) { + throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + } + return new ProvisionResponse(deviceCredentialsService.findDeviceCredentialsByDeviceId(device.getTenantId(), device.getId()), ProvisionResponseStatus.SUCCESS); + } + + private ProvisionResponse createDevice(ProvisionRequest provisionRequest, DeviceProfile profile) { + deviceCreationLock.lock(); + try { + return processCreateDevice(provisionRequest, profile); + } finally { + deviceCreationLock.unlock(); + } + } + + private void notify(Device device, ProvisionRequest provisionRequest, String type, boolean success) { + pushProvisionEventToRuleEngine(provisionRequest, device, type); + logAction(device.getTenantId(), device.getCustomerId(), device, success, provisionRequest); + } + + private ProvisionResponse processCreateDevice(ProvisionRequest provisionRequest, DeviceProfile profile) { + Device device = deviceService.findDeviceByTenantIdAndName(profile.getTenantId(), provisionRequest.getDeviceName()); + try { + if (device == null) { + if (StringUtils.isEmpty(provisionRequest.getDeviceName())) { + String newDeviceName = RandomStringUtils.randomAlphanumeric(20); + log.info("Device name not found in provision request. Generated name is: {}", newDeviceName); + provisionRequest.setDeviceName(newDeviceName); + } + Device savedDevice = deviceService.saveDevice(provisionRequest, profile); + + deviceStateService.onDeviceAdded(savedDevice); + saveProvisionStateAttribute(savedDevice).get(); + pushDeviceCreatedEventToRuleEngine(savedDevice); + notify(savedDevice, provisionRequest, DataConstants.PROVISION_SUCCESS, true); + + return new ProvisionResponse(getDeviceCredentials(savedDevice), ProvisionResponseStatus.SUCCESS); + } else { + log.warn("[{}] The device is already provisioned!", device.getName()); + notify(device, provisionRequest, DataConstants.PROVISION_FAILURE, false); + throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + } + } catch (InterruptedException | ExecutionException e) { + throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + } + } + + private ListenableFuture> saveProvisionStateAttribute(Device device) { + return attributesService.save(device.getTenantId(), device.getId(), DataConstants.SERVER_SCOPE, + Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry(DEVICE_PROVISION_STATE, PROVISIONED_STATE), + System.currentTimeMillis()))); + } + + private DeviceCredentials getDeviceCredentials(Device device) { + return deviceCredentialsService.findDeviceCredentialsByDeviceId(device.getTenantId(), device.getId()); + } + + private void pushProvisionEventToRuleEngine(ProvisionRequest request, Device device, String type) { + try { + JsonNode entityNode = JacksonUtil.valueToTree(request); + TbMsg msg = TbMsg.newMsg(type, device.getId(), createTbMsgMetaData(device), JacksonUtil.toString(entityNode)); + sendToRuleEngine(device.getTenantId(), msg, null); + } catch (IllegalArgumentException e) { + log.warn("[{}] Failed to push device action to rule engine: {}", device.getId(), type, e); + } + } + + private void pushDeviceCreatedEventToRuleEngine(Device device) { + try { + ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.valueToTree(device); + TbMsg msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, device.getId(), createTbMsgMetaData(device), JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode)); + sendToRuleEngine(device.getTenantId(), msg, null); + } catch (JsonProcessingException | IllegalArgumentException e) { + log.warn("[{}] Failed to push device action to rule engine: {}", device.getId(), DataConstants.ENTITY_CREATED, e); + } + } + + protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); + ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); + } + + private TbMsgMetaData createTbMsgMetaData(Device device) { + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("tenantId", device.getTenantId().toString()); + return metaData; + } + + private void logAction(TenantId tenantId, CustomerId customerId, Device device, boolean success, ProvisionRequest provisionRequest) { + ActionType actionType = success ? ActionType.PROVISION_SUCCESS : ActionType.PROVISION_FAILURE; + auditLogService.logEntityAction(tenantId, customerId, new UserId(UserId.NULL_UUID), device.getName(), device.getId(), device, actionType, null, provisionRequest); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index bb875ec5c4..fe1b06399c 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -28,11 +28,21 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.device.profile.AlarmCondition; +import org.thingsboard.server.common.data.device.profile.AlarmRule; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.device.profile.DisabledDeviceProfileProvisionConfiguration; +import org.thingsboard.server.common.data.device.profile.SimpleAlarmConditionSpec; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -41,19 +51,30 @@ import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate; +import org.thingsboard.server.common.data.query.DynamicValue; +import org.thingsboard.server.common.data.query.DynamicValueSourceType; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.UserCredentials; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; @@ -61,6 +82,8 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import java.util.Arrays; +import java.util.Collections; +import java.util.TreeMap; @Service @Profile("install") @@ -95,12 +118,6 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Autowired private CustomerService customerService; - @Autowired - private RelationService relationService; - - @Autowired - private AssetService assetService; - @Autowired private DeviceService deviceService; @@ -113,6 +130,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Autowired private DeviceCredentialsService deviceCredentialsService; + @Autowired + private RuleChainService ruleChainService; + @Bean protected BCryptPasswordEncoder passwordEncoder() { return new BCryptPasswordEncoder(); @@ -127,13 +147,16 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { public void createDefaultTenantProfiles() throws Exception { tenantProfileService.findOrCreateDefaultTenantProfile(TenantId.SYS_TENANT_ID); + TenantProfileData tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + TenantProfile isolatedTbCoreProfile = new TenantProfile(); isolatedTbCoreProfile.setDefault(false); isolatedTbCoreProfile.setName("Isolated TB Core"); - isolatedTbCoreProfile.setProfileData(new TenantProfileData()); isolatedTbCoreProfile.setDescription("Isolated TB Core tenant profile"); isolatedTbCoreProfile.setIsolatedTbCore(true); isolatedTbCoreProfile.setIsolatedTbRuleEngine(false); + isolatedTbCoreProfile.setProfileData(tenantProfileData); try { tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreProfile); } catch (DataValidationException e) { @@ -143,10 +166,11 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { TenantProfile isolatedTbRuleEngineProfile = new TenantProfile(); isolatedTbRuleEngineProfile.setDefault(false); isolatedTbRuleEngineProfile.setName("Isolated TB Rule Engine"); - isolatedTbRuleEngineProfile.setProfileData(new TenantProfileData()); isolatedTbRuleEngineProfile.setDescription("Isolated TB Rule Engine tenant profile"); isolatedTbRuleEngineProfile.setIsolatedTbCore(false); isolatedTbRuleEngineProfile.setIsolatedTbRuleEngine(true); + isolatedTbRuleEngineProfile.setProfileData(tenantProfileData); + try { tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbRuleEngineProfile); } catch (DataValidationException e) { @@ -156,10 +180,11 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { TenantProfile isolatedTbCoreAndTbRuleEngineProfile = new TenantProfile(); isolatedTbCoreAndTbRuleEngineProfile.setDefault(false); isolatedTbCoreAndTbRuleEngineProfile.setName("Isolated TB Core and TB Rule Engine"); - isolatedTbCoreAndTbRuleEngineProfile.setProfileData(new TenantProfileData()); isolatedTbCoreAndTbRuleEngineProfile.setDescription("Isolated TB Core and TB Rule Engine tenant profile"); isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbCore(true); isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbRuleEngine(true); + isolatedTbCoreAndTbRuleEngineProfile.setProfileData(tenantProfileData); + try { tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreAndTbRuleEngineProfile); } catch (DataValidationException e) { @@ -173,6 +198,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { generalSettings.setKey("general"); ObjectNode node = objectMapper.createObjectNode(); node.put("baseUrl", "http://localhost:8080"); + node.put("prohibitDifferentUrl", true); generalSettings.setJsonValue(node); adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings); @@ -193,6 +219,11 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, mailSettings); } + @Override + public void createOAuth2Templates() throws Exception { + installScripts.createOAuth2Templates(); + } + @Override public void loadDemoData() throws Exception { Tenant demoTenant = new Tenant(); @@ -233,35 +264,149 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { createDevice(demoTenant.getId(), null, defaultDeviceProfile.getId(), "Raspberry Pi Demo Device", "RASPBERRY_PI_DEMO_TOKEN", "Demo device that is used in " + "Raspberry Pi GPIO control sample application"); - Asset thermostatAlarms = new Asset(); - thermostatAlarms.setTenantId(demoTenant.getId()); - thermostatAlarms.setName("Thermostat Alarms"); - thermostatAlarms.setType("AlarmPropagationAsset"); - thermostatAlarms = assetService.saveAsset(thermostatAlarms); - - DeviceProfile thermostatDeviceProfile = this.deviceProfileService.findOrCreateDeviceProfile(demoTenant.getId(), "thermostat"); - - DeviceId t1Id = createDevice(demoTenant.getId(), null, thermostatDeviceProfile.getId(), "Thermostat T1", "T1_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); - DeviceId t2Id = createDevice(demoTenant.getId(), null, thermostatDeviceProfile.getId(), "Thermostat T2", "T2_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); - - relationService.saveRelation(thermostatAlarms.getTenantId(), new EntityRelation(thermostatAlarms.getId(), t1Id, "ToAlarmPropagationAsset")); - relationService.saveRelation(thermostatAlarms.getTenantId(), new EntityRelation(thermostatAlarms.getId(), t2Id, "ToAlarmPropagationAsset")); + DeviceProfile thermostatDeviceProfile = new DeviceProfile(); + thermostatDeviceProfile.setTenantId(demoTenant.getId()); + thermostatDeviceProfile.setDefault(false); + thermostatDeviceProfile.setName("thermostat"); + thermostatDeviceProfile.setType(DeviceProfileType.DEFAULT); + thermostatDeviceProfile.setTransportType(DeviceTransportType.DEFAULT); + thermostatDeviceProfile.setProvisionType(DeviceProfileProvisionType.DISABLED); + thermostatDeviceProfile.setDescription("Thermostat device profile"); + thermostatDeviceProfile.setDefaultRuleChainId(ruleChainService.findTenantRuleChainsByType( + demoTenant.getId(), RuleChainType.CORE, new PageLink(1, 0, "Thermostat")).getData().get(0).getId()); + + DeviceProfileData deviceProfileData = new DeviceProfileData(); + DefaultDeviceProfileConfiguration configuration = new DefaultDeviceProfileConfiguration(); + DefaultDeviceProfileTransportConfiguration transportConfiguration = new DefaultDeviceProfileTransportConfiguration(); + DisabledDeviceProfileProvisionConfiguration provisionConfiguration = new DisabledDeviceProfileProvisionConfiguration(null); + deviceProfileData.setConfiguration(configuration); + deviceProfileData.setTransportConfiguration(transportConfiguration); + deviceProfileData.setProvisionConfiguration(provisionConfiguration); + thermostatDeviceProfile.setProfileData(deviceProfileData); + + DeviceProfileAlarm highTemperature = new DeviceProfileAlarm(); + highTemperature.setId("highTemperatureAlarmID"); + highTemperature.setAlarmType("High Temperature"); + AlarmRule temperatureRule = new AlarmRule(); + AlarmCondition temperatureCondition = new AlarmCondition(); + temperatureCondition.setSpec(new SimpleAlarmConditionSpec()); + + KeyFilter temperatureAlarmFlagAttributeFilter = new KeyFilter(); + temperatureAlarmFlagAttributeFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperatureAlarmFlag")); + temperatureAlarmFlagAttributeFilter.setValueType(EntityKeyValueType.BOOLEAN); + BooleanFilterPredicate temperatureAlarmFlagAttributePredicate = new BooleanFilterPredicate(); + temperatureAlarmFlagAttributePredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); + temperatureAlarmFlagAttributePredicate.setValue(new FilterPredicateValue<>(Boolean.TRUE)); + temperatureAlarmFlagAttributeFilter.setPredicate(temperatureAlarmFlagAttributePredicate); + + KeyFilter temperatureTimeseriesFilter = new KeyFilter(); + temperatureTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + temperatureTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); + NumericFilterPredicate temperatureTimeseriesFilterPredicate = new NumericFilterPredicate(); + temperatureTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + FilterPredicateValue temperatureTimeseriesPredicateValue = + new FilterPredicateValue<>(25.0, null, + new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "temperatureAlarmThreshold")); + temperatureTimeseriesFilterPredicate.setValue(temperatureTimeseriesPredicateValue); + temperatureTimeseriesFilter.setPredicate(temperatureTimeseriesFilterPredicate); + temperatureCondition.setCondition(Arrays.asList(temperatureAlarmFlagAttributeFilter, temperatureTimeseriesFilter)); + temperatureRule.setAlarmDetails("Current temperature = ${temperature}"); + temperatureRule.setCondition(temperatureCondition); + highTemperature.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.MAJOR, temperatureRule))); + + AlarmRule clearTemperatureRule = new AlarmRule(); + AlarmCondition clearTemperatureCondition = new AlarmCondition(); + clearTemperatureCondition.setSpec(new SimpleAlarmConditionSpec()); + + KeyFilter clearTemperatureTimeseriesFilter = new KeyFilter(); + clearTemperatureTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + clearTemperatureTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); + NumericFilterPredicate clearTemperatureTimeseriesFilterPredicate = new NumericFilterPredicate(); + clearTemperatureTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS_OR_EQUAL); + FilterPredicateValue clearTemperatureTimeseriesPredicateValue = + new FilterPredicateValue<>(25.0, null, + new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "temperatureAlarmThreshold")); + + clearTemperatureTimeseriesFilterPredicate.setValue(clearTemperatureTimeseriesPredicateValue); + clearTemperatureTimeseriesFilter.setPredicate(clearTemperatureTimeseriesFilterPredicate); + clearTemperatureCondition.setCondition(Collections.singletonList(clearTemperatureTimeseriesFilter)); + clearTemperatureRule.setCondition(clearTemperatureCondition); + clearTemperatureRule.setAlarmDetails("Current temperature = ${temperature}"); + highTemperature.setClearRule(clearTemperatureRule); + + DeviceProfileAlarm lowHumidity = new DeviceProfileAlarm(); + lowHumidity.setId("lowHumidityAlarmID"); + lowHumidity.setAlarmType("Low Humidity"); + AlarmRule humidityRule = new AlarmRule(); + AlarmCondition humidityCondition = new AlarmCondition(); + humidityCondition.setSpec(new SimpleAlarmConditionSpec()); + + KeyFilter humidityAlarmFlagAttributeFilter = new KeyFilter(); + humidityAlarmFlagAttributeFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "humidityAlarmFlag")); + humidityAlarmFlagAttributeFilter.setValueType(EntityKeyValueType.BOOLEAN); + BooleanFilterPredicate humidityAlarmFlagAttributePredicate = new BooleanFilterPredicate(); + humidityAlarmFlagAttributePredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); + humidityAlarmFlagAttributePredicate.setValue(new FilterPredicateValue<>(Boolean.TRUE)); + humidityAlarmFlagAttributeFilter.setPredicate(humidityAlarmFlagAttributePredicate); + + KeyFilter humidityTimeseriesFilter = new KeyFilter(); + humidityTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "humidity")); + humidityTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); + NumericFilterPredicate humidityTimeseriesFilterPredicate = new NumericFilterPredicate(); + humidityTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS); + FilterPredicateValue humidityTimeseriesPredicateValue = + new FilterPredicateValue<>(60.0, null, + new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "humidityAlarmThreshold")); + humidityTimeseriesFilterPredicate.setValue(humidityTimeseriesPredicateValue); + humidityTimeseriesFilter.setPredicate(humidityTimeseriesFilterPredicate); + humidityCondition.setCondition(Arrays.asList(humidityAlarmFlagAttributeFilter, humidityTimeseriesFilter)); + + humidityRule.setCondition(humidityCondition); + humidityRule.setAlarmDetails("Current humidity = ${humidity}"); + lowHumidity.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.MINOR, humidityRule))); + + AlarmRule clearHumidityRule = new AlarmRule(); + AlarmCondition clearHumidityCondition = new AlarmCondition(); + clearHumidityCondition.setSpec(new SimpleAlarmConditionSpec()); + + KeyFilter clearHumidityTimeseriesFilter = new KeyFilter(); + clearHumidityTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "humidity")); + clearHumidityTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); + NumericFilterPredicate clearHumidityTimeseriesFilterPredicate = new NumericFilterPredicate(); + clearHumidityTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER_OR_EQUAL); + FilterPredicateValue clearHumidityTimeseriesPredicateValue = + new FilterPredicateValue<>(60.0, null, + new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "humidityAlarmThreshold")); + + clearHumidityTimeseriesFilterPredicate.setValue(clearHumidityTimeseriesPredicateValue); + clearHumidityTimeseriesFilter.setPredicate(clearHumidityTimeseriesFilterPredicate); + clearHumidityCondition.setCondition(Collections.singletonList(clearHumidityTimeseriesFilter)); + clearHumidityRule.setCondition(clearHumidityCondition); + clearHumidityRule.setAlarmDetails("Current humidity = ${humidity}"); + lowHumidity.setClearRule(clearHumidityRule); + + deviceProfileData.setAlarms(Arrays.asList(highTemperature, lowHumidity)); + + DeviceProfile savedThermostatDeviceProfile = deviceProfileService.saveDeviceProfile(thermostatDeviceProfile); + + DeviceId t1Id = createDevice(demoTenant.getId(), null, savedThermostatDeviceProfile.getId(), "Thermostat T1", "T1_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); + DeviceId t2Id = createDevice(demoTenant.getId(), null, savedThermostatDeviceProfile.getId(), "Thermostat T2", "T2_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); attributesService.save(demoTenant.getId(), t1Id, DataConstants.SERVER_SCOPE, Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.3948)), new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -122.1503)), - new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)), - new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)), - new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 20)), - new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 50)))); + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("temperatureAlarmFlag", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("humidityAlarmFlag", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("temperatureAlarmThreshold", (long) 20)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("humidityAlarmThreshold", (long) 50)))); attributesService.save(demoTenant.getId(), t2Id, DataConstants.SERVER_SCOPE, Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.493801)), new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -121.948769)), - new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)), - new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)), - new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 25)), - new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 30)))); + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("temperatureAlarmFlag", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("humidityAlarmFlag", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("temperatureAlarmThreshold", (long) 25)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("humidityAlarmThreshold", (long) 30)))); installScripts.loadDashboards(demoTenant.getId(), null); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 7493a8ecd7..163d215f89 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -26,11 +26,13 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationTemplate; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -40,6 +42,7 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Optional; import static org.thingsboard.server.service.install.DatabaseHelper.objectMapper; @@ -61,6 +64,7 @@ public class InstallScripts { public static final String DEMO_DIR = "demo"; public static final String RULE_CHAINS_DIR = "rule_chains"; public static final String WIDGET_BUNDLES_DIR = "widget_bundles"; + public static final String OAUTH2_CONFIG_TEMPLATES_DIR = "oauth2_config_templates"; public static final String DASHBOARDS_DIR = "dashboards"; public static final String JSON_EXT = ".json"; @@ -80,6 +84,9 @@ public class InstallScripts { @Autowired private WidgetsBundleService widgetsBundleService; + @Autowired + private OAuth2ConfigTemplateService oAuth2TemplateService; + public Path getTenantRuleChainsDir() { return Paths.get(getDataDir(), JSON_DIR, TENANT_DIR, RULE_CHAINS_DIR); } @@ -146,12 +153,6 @@ public class InstallScripts { return ruleChain; } - - public void createDefaultEdgeRuleChains(TenantId tenantId) { - Path tenantChainsDir = getTenantRuleChainsDir(); - loadRuleChainFromFile(tenantId, tenantChainsDir.resolve("edge_root_rule_chain.json")); - } - public void loadSystemWidgets() throws Exception { Path widgetBundlesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_BUNDLES_DIR); try (DirectoryStream dirStream = Files.newDirectoryStream(widgetBundlesDir, path -> path.toString().endsWith(JSON_EXT))) { @@ -206,50 +207,42 @@ public class InstallScripts { } } - public void loadDemoRuleChains(TenantId tenantId) throws Exception { - Path ruleChainsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, RULE_CHAINS_DIR); try { - JsonNode ruleChainJson = objectMapper.readTree(ruleChainsDir.resolve("thermostat_alarms.json").toFile()); - RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); - RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); - ruleChain.setTenantId(tenantId); - ruleChain = ruleChainService.saveRuleChain(ruleChain); - ruleChainMetaData.setRuleChainId(ruleChain.getId()); - ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); - - JsonNode rootChainJson = objectMapper.readTree(ruleChainsDir.resolve("root_rule_chain.json").toFile()); - RuleChain rootChain = objectMapper.treeToValue(rootChainJson.get("ruleChain"), RuleChain.class); - RuleChainMetaData rootChainMetaData = objectMapper.treeToValue(rootChainJson.get("metadata"), RuleChainMetaData.class); - - RuleChainId thermostatsRuleChainId = ruleChain.getId(); - rootChainMetaData.getRuleChainConnections().forEach(connection -> connection.setTargetRuleChainId(thermostatsRuleChainId)); - rootChain.setTenantId(tenantId); - rootChain = ruleChainService.saveRuleChain(rootChain); - rootChainMetaData.setRuleChainId(rootChain.getId()); - ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), rootChainMetaData); - - loadRuleChainFromFile(tenantId, ruleChainsDir.resolve("edge_root_rule_chain.json")); + createDefaultRuleChains(tenantId); + createDefaultRuleChain(tenantId, "Thermostat"); + createDefaultEdgeRuleChains(tenantId); } catch (Exception e) { log.error("Unable to load dashboard from json", e); throw new RuntimeException("Unable to load dashboard from json", e); } } - private void loadRuleChainFromFile(TenantId tenantId, Path ruleChainPath) { - try { - JsonNode ruleChainJson = objectMapper.readTree(ruleChainPath.toFile()); - RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); - RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); - - ruleChain.setTenantId(tenantId); - ruleChain = ruleChainService.saveRuleChain(ruleChain); + public void createDefaultEdgeRuleChains(TenantId tenantId) throws IOException { + Path tenantChainsDir = getTenantRuleChainsDir(); + createRuleChainFromFile(tenantId, tenantChainsDir.resolve("edge_root_rule_chain.json"), null); + } - ruleChainMetaData.setRuleChainId(ruleChain.getId()); - ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); - } catch (Exception e) { - log.error("Unable to load rule chain from json: [{}]", ruleChainPath.toString()); - throw new RuntimeException("Unable to load rule chain from json", e); + public void createOAuth2Templates() throws Exception { + Path oauth2ConfigTemplatesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, OAUTH2_CONFIG_TEMPLATES_DIR); + try (DirectoryStream dirStream = Files.newDirectoryStream(oauth2ConfigTemplatesDir, path -> path.toString().endsWith(JSON_EXT))) { + dirStream.forEach( + path -> { + try { + JsonNode oauth2ConfigTemplateJson = objectMapper.readTree(path.toFile()); + OAuth2ClientRegistrationTemplate clientRegistrationTemplate = objectMapper.treeToValue(oauth2ConfigTemplateJson, OAuth2ClientRegistrationTemplate.class); + Optional existingClientRegistrationTemplate = + oAuth2TemplateService.findClientRegistrationTemplateByProviderId(clientRegistrationTemplate.getProviderId()); + if (existingClientRegistrationTemplate.isPresent()) { + clientRegistrationTemplate.setId(existingClientRegistrationTemplate.get().getId()); + } + oAuth2TemplateService.saveClientRegistrationTemplate(clientRegistrationTemplate); + } catch (Exception e) { + log.error("Unable to load oauth2 config templates from json: [{}]", path.toString()); + throw new RuntimeException("Unable to load oauth2 config templates from json", e); + } + } + ); } } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 383236af11..99bc1302df 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.service.install.sql.SqlDbHelper; import java.nio.charset.Charset; @@ -96,6 +97,9 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService @Autowired private DeviceProfileService deviceProfileService; + @Autowired + private ApiUsageStateService apiUsageStateService; + @Override public void upgradeDatabase(String fromVersion) throws Exception { @@ -364,6 +368,24 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } catch (Exception e) { } + try { + conn.createStatement().execute("CREATE TABLE IF NOT EXISTS api_usage_state (" + + " id uuid NOT NULL CONSTRAINT usage_record_pkey PRIMARY KEY," + + " created_time bigint NOT NULL," + + " tenant_id uuid," + + " entity_type varchar(32)," + + " entity_id uuid," + + " transport varchar(32)," + + " db_storage varchar(32)," + + " re_exec varchar(32)," + + " js_exec varchar(32)," + + " email_exec varchar(32)," + + " sms_exec varchar(32)," + + " CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id)\n" + + ");"); + } catch (Exception e) { + } + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.1.1", "schema_update_before.sql"); loadSql(schemaUpdateFile, conn); @@ -379,6 +401,10 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService do { pageData = tenantService.findTenants(pageLink); for (Tenant tenant : pageData.getData()) { + try { + apiUsageStateService.createDefaultApiUsageState(tenant.getId()); + } catch (Exception e) { + } List deviceTypes = deviceService.findDeviceTypesByTenantId(tenant.getId()).get(); try { deviceProfileService.createDefaultDeviceProfile(tenant.getId()); diff --git a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java index b588c2dff2..b1c670a14c 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java @@ -23,6 +23,8 @@ public interface SystemDataLoaderService { void createAdminSettings() throws Exception; + void createOAuth2Templates() throws Exception; + void loadSystemWidgets() throws Exception; void updateSystemWidgets() throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index c424f14f3b..20231cb01a 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -187,7 +187,7 @@ public class DefaultDataUpdateService implements DataUpdateService { ruleChainService.saveRuleChainMetaData(tenant.getId(), md); } } catch (Exception e) { - log.error("Unable to update Tenant", e); + log.error("[{}] Unable to update Tenant: {}", tenant.getId(), tenant.getName(), e); } } }; diff --git a/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java b/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java index bcf541880c..e9cef05dd5 100644 --- a/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java +++ b/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java @@ -20,8 +20,10 @@ import freemarker.template.Configuration; import freemarker.template.Template; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; +import org.springframework.context.annotation.Lazy; import org.springframework.core.NestedRuntimeException; import org.springframework.mail.javamail.JavaMailSenderImpl; import org.springframework.mail.javamail.MimeMessageHelper; @@ -29,12 +31,18 @@ import org.springframework.stereotype.Service; import org.springframework.ui.freemarker.FreeMarkerTemplateUtils; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.ApiFeature; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageStateMailMessage; +import org.thingsboard.server.common.data.ApiUsageStateValue; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import javax.annotation.PostConstruct; import javax.mail.MessagingException; @@ -51,18 +59,28 @@ public class DefaultMailService implements MailService { public static final String MAIL_PROP = "mail."; public static final String TARGET_EMAIL = "targetEmail"; public static final String UTF_8 = "UTF-8"; - @Autowired - private MessageSource messages; + public static final int _10K = 10000; + public static final int _1M = 1000000; + + private final MessageSource messages; + private final Configuration freemarkerConfig; + private final AdminSettingsService adminSettingsService; + private final TbApiUsageClient apiUsageClient; + @Lazy @Autowired - private Configuration freemarkerConfig; + private TbApiUsageStateService apiUsageStateService; private JavaMailSenderImpl mailSender; private String mailFrom; - @Autowired - private AdminSettingsService adminSettingsService; + public DefaultMailService(MessageSource messages, Configuration freemarkerConfig, AdminSettingsService adminSettingsService, TbApiUsageClient apiUsageClient) { + this.messages = messages; + this.freemarkerConfig = freemarkerConfig; + this.adminSettingsService = adminSettingsService; + this.apiUsageClient = apiUsageClient; + } @PostConstruct private void init() { @@ -141,7 +159,7 @@ public class DefaultMailService implements MailService { } @Override - public void sendEmail(String email, String subject, String message) throws ThingsboardException { + public void sendEmail(TenantId tenantId, String email, String subject, String message) throws ThingsboardException { sendMail(mailSender, mailFrom, email, subject, message); } @@ -216,20 +234,25 @@ public class DefaultMailService implements MailService { } @Override - public void send(String from, String to, String cc, String bcc, String subject, String body) throws MessagingException { - MimeMessage mailMsg = mailSender.createMimeMessage(); - MimeMessageHelper helper = new MimeMessageHelper(mailMsg, "UTF-8"); - helper.setFrom(StringUtils.isBlank(from) ? mailFrom : from); - helper.setTo(to.split("\\s*,\\s*")); - if (!StringUtils.isBlank(cc)) { - helper.setCc(cc.split("\\s*,\\s*")); - } - if (!StringUtils.isBlank(bcc)) { - helper.setBcc(bcc.split("\\s*,\\s*")); + public void send(TenantId tenantId, String from, String to, String cc, String bcc, String subject, String body) throws MessagingException { + if (apiUsageStateService.getApiUsageState(tenantId).isEmailSendEnabled()) { + MimeMessage mailMsg = mailSender.createMimeMessage(); + MimeMessageHelper helper = new MimeMessageHelper(mailMsg, "UTF-8"); + helper.setFrom(StringUtils.isBlank(from) ? mailFrom : from); + helper.setTo(to.split("\\s*,\\s*")); + if (!StringUtils.isBlank(cc)) { + helper.setCc(cc.split("\\s*,\\s*")); + } + if (!StringUtils.isBlank(bcc)) { + helper.setBcc(bcc.split("\\s*,\\s*")); + } + helper.setSubject(subject); + helper.setText(body); + mailSender.send(helper.getMimeMessage()); + apiUsageClient.report(tenantId, ApiUsageRecordKey.EMAIL_EXEC_COUNT, 1); + } else { + throw new RuntimeException("Email sending is disabled due to API limits!"); } - helper.setSubject(subject); - helper.setText(body); - mailSender.send(helper.getMimeMessage()); } @Override @@ -246,6 +269,122 @@ public class DefaultMailService implements MailService { sendMail(mailSender, mailFrom, email, subject, message); } + @Override + public void sendApiFeatureStateEmail(ApiFeature apiFeature, ApiUsageStateValue stateValue, String email, ApiUsageStateMailMessage msg) throws ThingsboardException { + String subject = messages.getMessage("api.usage.state", null, Locale.US); + + Map model = new HashMap<>(); + model.put("apiFeature", apiFeature.getLabel()); + model.put(TARGET_EMAIL, email); + + String message = null; + + switch (stateValue) { + case ENABLED: + model.put("apiLabel", toEnabledValueLabel(apiFeature)); + message = mergeTemplateIntoString("state.enabled.ftl", model); + break; + case WARNING: + model.put("apiValueLabel", toDisabledValueLabel(apiFeature) + " " + toWarningValueLabel(msg.getKey(), msg.getValue(), msg.getThreshold())); + message = mergeTemplateIntoString("state.warning.ftl", model); + break; + case DISABLED: + model.put("apiLimitValueLabel", toDisabledValueLabel(apiFeature) + " " + toDisabledValueLabel(msg.getKey(), msg.getThreshold())); + message = mergeTemplateIntoString("state.disabled.ftl", model); + break; + } + sendMail(mailSender, mailFrom, email, subject, message); + } + + private String toEnabledValueLabel(ApiFeature apiFeature) { + switch (apiFeature) { + case DB: + return "save"; + case TRANSPORT: + return "receive"; + case JS: + return "invoke"; + case RE: + return "process"; + case EMAIL: + case SMS: + return "send"; + default: + throw new RuntimeException("Not implemented!"); + } + } + + private String toDisabledValueLabel(ApiFeature apiFeature) { + switch (apiFeature) { + case DB: + return "saved"; + case TRANSPORT: + return "received"; + case JS: + return "invoked"; + case RE: + return "processed"; + case EMAIL: + case SMS: + return "sent"; + default: + throw new RuntimeException("Not implemented!"); + } + } + + private String toWarningValueLabel(ApiUsageRecordKey key, long value, long threshold) { + String valueInM = getValueAsString(value); + String thresholdInM = getValueAsString(threshold); + switch (key) { + case STORAGE_DP_COUNT: + case TRANSPORT_DP_COUNT: + return valueInM + " out of " + thresholdInM + " allowed data points"; + case TRANSPORT_MSG_COUNT: + return valueInM + " out of " + thresholdInM + " allowed messages"; + case JS_EXEC_COUNT: + return valueInM + " out of " + thresholdInM + " allowed JavaScript functions"; + case RE_EXEC_COUNT: + return valueInM + " out of " + thresholdInM + " allowed Rule Engine messages"; + case EMAIL_EXEC_COUNT: + return valueInM + " out of " + thresholdInM + " allowed Email messages"; + case SMS_EXEC_COUNT: + return valueInM + " out of " + thresholdInM + " allowed SMS messages"; + default: + throw new RuntimeException("Not implemented!"); + } + } + + private String toDisabledValueLabel(ApiUsageRecordKey key, long value) { + switch (key) { + case STORAGE_DP_COUNT: + case TRANSPORT_DP_COUNT: + return getValueAsString(value) + " data points"; + case TRANSPORT_MSG_COUNT: + return getValueAsString(value) + " messages"; + case JS_EXEC_COUNT: + return "JavaScript functions " + getValueAsString(value) + " times"; + case RE_EXEC_COUNT: + return getValueAsString(value) + " Rule Engine messages"; + case EMAIL_EXEC_COUNT: + return getValueAsString(value) + " Email messages"; + case SMS_EXEC_COUNT: + return getValueAsString(value) + " SMS messages"; + default: + throw new RuntimeException("Not implemented!"); + } + } + + @NotNull + private String getValueAsString(long value) { + if (value > _1M && value % _1M < _10K) { + return value / _1M + "M"; + } else if (value > _10K) { + return String.format("%.2fM", ((double) value) / 1000000); + } else { + return value + ""; + } + } + private void sendMail(JavaMailSenderImpl mailSender, String mailFrom, String email, String subject, String message) throws ThingsboardException { diff --git a/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java b/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java index b0b47fe886..3db23c65e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; @@ -29,6 +30,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiConsumer; +import java.util.function.Consumer; @Service @Slf4j @@ -40,6 +43,8 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { private final ConcurrentMap deviceProfilesMap = new ConcurrentHashMap<>(); private final ConcurrentMap devicesMap = new ConcurrentHashMap<>(); + private final ConcurrentMap>> profileListeners = new ConcurrentHashMap<>(); + private final ConcurrentMap>> deviceProfileListeners = new ConcurrentHashMap<>(); public DefaultTbDeviceProfileCache(DeviceProfileService deviceProfileService, DeviceService deviceService) { this.deviceProfileService = deviceProfileService; @@ -50,19 +55,21 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { public DeviceProfile get(TenantId tenantId, DeviceProfileId deviceProfileId) { DeviceProfile profile = deviceProfilesMap.get(deviceProfileId); if (profile == null) { - profile = deviceProfilesMap.get(deviceProfileId); - if (profile == null) { - deviceProfileFetchLock.lock(); - try { + deviceProfileFetchLock.lock(); + try { + profile = deviceProfilesMap.get(deviceProfileId); + if (profile == null) { profile = deviceProfileService.findDeviceProfileById(tenantId, deviceProfileId); if (profile != null) { deviceProfilesMap.put(deviceProfileId, profile); + log.debug("[{}] Fetch device profile into cache: {}", profile.getId(), profile); } - } finally { - deviceProfileFetchLock.unlock(); } + } finally { + deviceProfileFetchLock.unlock(); } } + log.trace("[{}] Found device profile in cache: {}", deviceProfileId, profile); return profile; } @@ -74,26 +81,82 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { if (device != null) { profileId = device.getDeviceProfileId(); devicesMap.put(deviceId, profileId); + } else { + return null; } } return get(tenantId, profileId); } @Override - public void put(DeviceProfile profile) { - if (profile.getId() != null) { - deviceProfilesMap.put(profile.getId(), profile); + public void evict(TenantId tenantId, DeviceProfileId profileId) { + DeviceProfile oldProfile = deviceProfilesMap.remove(profileId); + log.debug("[{}] evict device profile from cache: {}", profileId, oldProfile); + DeviceProfile newProfile = get(tenantId, profileId); + if (newProfile != null) { + notifyProfileListeners(newProfile); } } @Override - public void evict(DeviceProfileId profileId) { - deviceProfilesMap.remove(profileId); + public void evict(TenantId tenantId, DeviceId deviceId) { + DeviceProfileId old = devicesMap.remove(deviceId); + if (old != null) { + DeviceProfile newProfile = get(tenantId, deviceId); + if (newProfile == null || !old.equals(newProfile.getId())) { + notifyDeviceListeners(tenantId, deviceId, newProfile); + } + } + } + + @Override + public void addListener(TenantId tenantId, EntityId listenerId, + Consumer profileListener, + BiConsumer deviceListener) { + if (profileListener != null) { + profileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, profileListener); + } + if (deviceListener != null) { + deviceProfileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, deviceListener); + } + } + + @Override + public DeviceProfile find(DeviceProfileId deviceProfileId) { + return deviceProfileService.findDeviceProfileById(TenantId.SYS_TENANT_ID, deviceProfileId); + } + + @Override + public DeviceProfile findOrCreateDeviceProfile(TenantId tenantId, String profileName) { + return deviceProfileService.findOrCreateDeviceProfile(tenantId, profileName); } @Override - public void evict(DeviceId deviceId) { - devicesMap.remove(deviceId); + public void removeListener(TenantId tenantId, EntityId listenerId) { + ConcurrentMap> tenantListeners = profileListeners.get(tenantId); + if (tenantListeners != null) { + tenantListeners.remove(listenerId); + } + ConcurrentMap> deviceListeners = deviceProfileListeners.get(tenantId); + if (deviceListeners != null) { + deviceListeners.remove(listenerId); + } + } + + private void notifyProfileListeners(DeviceProfile profile) { + ConcurrentMap> tenantListeners = profileListeners.get(profile.getTenantId()); + if (tenantListeners != null) { + tenantListeners.forEach((id, listener) -> listener.accept(profile)); + } + } + + private void notifyDeviceListeners(TenantId tenantId, DeviceId deviceId, DeviceProfile profile) { + if (profile != null) { + ConcurrentMap> tenantListeners = deviceProfileListeners.get(tenantId); + if (tenantListeners != null) { + tenantListeners.forEach((id, listener) -> listener.accept(deviceId, profile)); + } + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java b/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java index ec19eb1da6..10e28bd299 100644 --- a/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java @@ -19,13 +19,15 @@ import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; public interface TbDeviceProfileCache extends RuleEngineDeviceProfileCache { - void put(DeviceProfile profile); + void evict(TenantId tenantId, DeviceProfileId id); - void evict(DeviceProfileId id); + void evict(TenantId tenantId, DeviceId id); - void evict(DeviceId id); + DeviceProfile find(DeviceProfileId deviceProfileId); + DeviceProfile findOrCreateDeviceProfile(TenantId tenantId, String deviceType); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 28cd998e46..a8a29f09e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -21,8 +21,13 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -43,6 +48,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; @@ -152,8 +158,16 @@ public class DefaultTbClusterService implements TbClusterService { private TbMsg transformMsg(TbMsg tbMsg, DeviceProfile deviceProfile) { if (deviceProfile != null) { RuleChainId targetRuleChainId = deviceProfile.getDefaultRuleChainId(); - if (targetRuleChainId != null && !targetRuleChainId.equals(tbMsg.getRuleChainId())) { + String targetQueueName = deviceProfile.getDefaultQueueName(); + boolean isRuleChainTransform = targetRuleChainId != null && !targetRuleChainId.equals(tbMsg.getRuleChainId()); + boolean isQueueTransform = targetQueueName != null && !targetQueueName.equals(tbMsg.getQueueName()); + + if (isRuleChainTransform && isQueueTransform) { + tbMsg = TbMsg.transformMsg(tbMsg, targetRuleChainId, targetQueueName); + } else if (isRuleChainTransform) { tbMsg = TbMsg.transformMsg(tbMsg, targetRuleChainId); + } else if (isQueueTransform) { + tbMsg = TbMsg.transformMsg(tbMsg, targetQueueName); } } return tbMsg; @@ -189,30 +203,78 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onDeviceProfileChange(DeviceProfile deviceProfile, TbQueueCallback callback) { - log.trace("[{}][{}] Processing device profile [{}] change event", deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile.getName()); - TransportProtos.DeviceProfileUpdateMsg profileUpdateMsg = TransportProtos.DeviceProfileUpdateMsg.newBuilder() - .setData(ByteString.copyFrom(encodingService.encode(deviceProfile))).build(); - ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setDeviceProfileUpdateMsg(profileUpdateMsg).build(); - broadcast(transportMsg); + onEntityChange(deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile, callback); + } + + @Override + public void onTenantProfileChange(TenantProfile tenantProfile, TbQueueCallback callback) { + onEntityChange(TenantId.SYS_TENANT_ID, tenantProfile.getId(), tenantProfile, callback); + } + + @Override + public void onTenantChange(Tenant tenant, TbQueueCallback callback) { + onEntityChange(TenantId.SYS_TENANT_ID, tenant.getId(), tenant, callback); + } + + @Override + public void onApiStateChange(ApiUsageState apiUsageState, TbQueueCallback callback) { + onEntityChange(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState, callback); + broadcast(new ComponentLifecycleMsg(apiUsageState.getTenantId(), apiUsageState.getId(), ComponentLifecycleEvent.UPDATED)); + } + + @Override + public void onDeviceProfileDelete(DeviceProfile entity, TbQueueCallback callback) { + onEntityDelete(entity.getTenantId(), entity.getId(), entity.getName(), callback); } @Override - public void onDeviceProfileDelete(DeviceProfile deviceProfile, TbQueueCallback callback) { - log.trace("[{}][{}] Processing device profile [{}] delete event", deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile.getName()); - TransportProtos.DeviceProfileDeleteMsg profileDeleteMsg = TransportProtos.DeviceProfileDeleteMsg.newBuilder() - .setProfileIdMSB(deviceProfile.getId().getId().getMostSignificantBits()) - .setProfileIdLSB(deviceProfile.getId().getId().getLeastSignificantBits()) + public void onTenantProfileDelete(TenantProfile entity, TbQueueCallback callback) { + onEntityDelete(TenantId.SYS_TENANT_ID, entity.getId(), entity.getName(), callback); + } + + @Override + public void onTenantDelete(Tenant entity, TbQueueCallback callback) { + onEntityDelete(TenantId.SYS_TENANT_ID, entity.getId(), entity.getName(), callback); + } + + @Override + public void onDeviceChange(Device entity, TbQueueCallback callback) { + onEntityChange(entity.getTenantId(), entity.getId(), entity, callback); + } + + @Override + public void onDeviceDeleted(Device entity, TbQueueCallback callback) { + onEntityDelete(entity.getTenantId(), entity.getId(), entity.getName(), callback); + } + + public void onEntityChange(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) { + String entityName = (entity instanceof HasName) ? ((HasName) entity).getName() : entity.getClass().getName(); + log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entityName); + TransportProtos.EntityUpdateMsg entityUpdateMsg = TransportProtos.EntityUpdateMsg.newBuilder() + .setEntityType(entityid.getEntityType().name()) + .setData(ByteString.copyFrom(encodingService.encode(entity))).build(); + ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityUpdateMsg(entityUpdateMsg).build(); + broadcast(transportMsg, callback); + } + + private void onEntityDelete(TenantId tenantId, EntityId entityId, String name, TbQueueCallback callback) { + log.trace("[{}][{}][{}] Processing [{}] delete event", tenantId, entityId.getEntityType(), entityId.getId(), name); + TransportProtos.EntityDeleteMsg entityDeleteMsg = TransportProtos.EntityDeleteMsg.newBuilder() + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) .build(); - ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setDeviceProfileDeleteMsg(profileDeleteMsg).build(); - broadcast(transportMsg); + ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityDeleteMsg(entityDeleteMsg).build(); + broadcast(transportMsg, callback); } - private void broadcast(ToTransportMsg transportMsg) { + private void broadcast(ToTransportMsg transportMsg, TbQueueCallback callback) { TbQueueProducer> toTransportNfProducer = producerProvider.getTransportNotificationsMsgProducer(); Set tbTransportServices = partitionService.getAllServiceIds(ServiceType.TB_TRANSPORT); + TbQueueCallback proxyCallback = callback != null ? new MultipleTbQueueCallbackWrapper(tbTransportServices.size(), callback) : null; for (String transportServiceId : tbTransportServices) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId); - toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), null); + toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), proxyCallback); toTransportNfs.incrementAndGet(); } } @@ -221,7 +283,10 @@ public class DefaultTbClusterService implements TbClusterService { byte[] msgBytes = encodingService.encode(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); - if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)) { + if (msg.getEntityId().getEntityType().equals(EntityType.TENANT) + || msg.getEntityId().getEntityType().equals(EntityType.TENANT_PROFILE) + || msg.getEntityId().getEntityType().equals(EntityType.DEVICE_PROFILE) + || msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index aa154b0525..f1e0ea719d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -15,10 +15,16 @@ */ package org.thingsboard.server.service.queue; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.alarm.Alarm; @@ -43,14 +49,17 @@ import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseP import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.edge.EdgeNotificationService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; @@ -69,6 +78,9 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @@ -87,35 +99,60 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> mainConsumer; private final DeviceStateService stateService; + private final TbApiUsageStateService statsService; private final TbLocalSubscriptionService localSubscriptionService; private final SubscriptionManagerService subscriptionManagerService; private final TbCoreDeviceRpcService tbCoreDeviceRpcService; private final EdgeNotificationService edgeNotificationService; private final TbCoreConsumerStats stats; + protected final TbQueueConsumer> usageStatsConsumer; - public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, - DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, - SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, - TbCoreDeviceRpcService tbCoreDeviceRpcService, StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache, + protected volatile ExecutorService usageStatsExecutor; + + public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, + ActorSystemContext actorContext, + DeviceStateService stateService, + TbLocalSubscriptionService localSubscriptionService, + SubscriptionManagerService subscriptionManagerService, + DataDecodingEncodingService encodingService, + TbCoreDeviceRpcService tbCoreDeviceRpcService, + StatsFactory statsFactory, + TbDeviceProfileCache deviceProfileCache, + TbApiUsageStateService statsService, + TbTenantProfileCache tenantProfileCache, + TbApiUsageStateService apiUsageStateService, EdgeNotificationService edgeNotificationService) { - super(actorContext, encodingService, deviceProfileCache, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); + super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); + this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer(); this.stateService = stateService; this.localSubscriptionService = localSubscriptionService; this.subscriptionManagerService = subscriptionManagerService; this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; this.edgeNotificationService = edgeNotificationService; this.stats = new TbCoreConsumerStats(statsFactory); + this.statsService = statsService; } @PostConstruct public void init() { super.init("tb-core-consumer", "tb-core-notifications-consumer"); + this.usageStatsExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer")); } @PreDestroy public void destroy() { super.destroy(); + if (usageStatsExecutor != null) { + usageStatsExecutor.shutdownNow(); + } + } + + @EventListener(ApplicationReadyEvent.class) + @Order(value = 2) + public void onApplicationEvent(ApplicationReadyEvent event) { + super.onApplicationEvent(event); + launchUsageStatsConsumer(); } @Override @@ -123,6 +160,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService tpi.newByTopic(usageStatsConsumer.getTopic())) + .collect(Collectors.toSet())); } } @@ -140,42 +183,51 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - pendingMap.forEach((id, msg) -> { - log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); - TbCallback callback = new TbPackCallback<>(id, ctx); - try { - ToCoreMsg toCoreMsg = msg.getValue(); - if (toCoreMsg.hasToSubscriptionMgrMsg()) { - log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); - forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); - } else if (toCoreMsg.hasToDeviceActorMsg()) { - log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); - forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); - } else if (toCoreMsg.hasDeviceStateServiceMsg()) { - log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); - forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); - } else if (toCoreMsg.hasEdgeNotificationMsg()) { - log.trace("[{}] Forwarding message to edge service {}", id, toCoreMsg.getEdgeNotificationMsg()); - forwardToEdgeNotificationService(toCoreMsg.getEdgeNotificationMsg(), callback); - } else if (toCoreMsg.getToDeviceActorNotificationMsg() != null && !toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); - if (actorMsg.isPresent()) { - TbActorMsg tbActorMsg = actorMsg.get(); - if (tbActorMsg.getMsgType().equals(MsgType.DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG)) { - tbCoreDeviceRpcService.forwardRpcRequestToDeviceActor((ToDeviceRpcRequestActorMsg) tbActorMsg); - } else { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.tell(actorMsg.get()); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); + Future> packSubmitFuture = consumersExecutor.submit(() -> { + pendingMap.forEach((id, msg) -> { + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + ToCoreMsg toCoreMsg = msg.getValue(); + pendingMsgHolder.setToCoreMsg(toCoreMsg); + if (toCoreMsg.hasToSubscriptionMgrMsg()) { + log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); + forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); + } else if (toCoreMsg.hasToDeviceActorMsg()) { + log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); + forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); + } else if (toCoreMsg.hasDeviceStateServiceMsg()) { + log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); + forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); + } else if (toCoreMsg.hasEdgeNotificationMsg()) { + log.trace("[{}] Forwarding message to edge service {}", id, toCoreMsg.getEdgeNotificationMsg()); + forwardToEdgeNotificationService(toCoreMsg.getEdgeNotificationMsg(), callback); + } else if (toCoreMsg.getToDeviceActorNotificationMsg() != null && !toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); + if (actorMsg.isPresent()) { + TbActorMsg tbActorMsg = actorMsg.get(); + if (tbActorMsg.getMsgType().equals(MsgType.DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG)) { + tbCoreDeviceRpcService.forwardRpcRequestToDeviceActor((ToDeviceRpcRequestActorMsg) tbActorMsg); + } else { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.tell(actorMsg.get()); + } } + callback.onSuccess(); } - callback.onSuccess(); + } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); + callback.onFailure(e); } - } catch (Throwable e) { - log.warn("[{}] Failed to process message: {}", id, msg, e); - callback.onFailure(e); - } + }); }); if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + if (!packSubmitFuture.isDone()) { + packSubmitFuture.cancel(true); + ToCoreMsg lastSubmitMsg = pendingMsgHolder.getToCoreMsg(); + log.info("Timeout to process message: {}", lastSubmitMsg); + } ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } @@ -195,6 +247,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService { + while (!stopped) { + try { + List> msgs = usageStatsConsumer.poll(getNotificationPollDuration()); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + pendingMap.forEach((id, msg) -> { + log.trace("[{}] Creating usage stats callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + handleUsageStats(msg, callback); + } catch (Throwable e) { + log.warn("[{}] Failed to process usage stats: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process usage stats: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process usage stats: {}", id, msg.getValue())); + } + usageStatsConsumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain usage stats from queue.", e); + try { + Thread.sleep(getNotificationPollDuration()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new usage stats", e2); + } + } + } + } + log.info("TB Usage Stats Consumer stopped."); + }); + } + + private void handleUsageStats(TbProtoQueueMsg msg, TbCallback callback) { + statsService.process(msg, callback); + } + private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) @@ -333,6 +438,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> nfConsumer; public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, - TbDeviceProfileCache deviceProfileCache, TbQueueConsumer> nfConsumer) { + TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache, TbApiUsageStateService apiUsageStateService, TbQueueConsumer> nfConsumer) { this.actorContext = actorContext; this.encodingService = encodingService; + this.tenantProfileCache = tenantProfileCache; this.deviceProfileCache = deviceProfileCache; + this.apiUsageStateService = apiUsageStateService; this.nfConsumer = nfConsumer; } @@ -77,6 +86,7 @@ public abstract class AbstractConsumerService scriptIdToNameMap = new ConcurrentHashMap<>(); - protected Map blackListedFunctions = new ConcurrentHashMap<>(); + protected Map disabledFunctions = new ConcurrentHashMap<>(); + + protected AbstractJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) { + this.apiUsageStateService = apiUsageStateService; + this.apiUsageClient = apiUsageClient; + } public void init(long maxRequestsTimeout) { if (maxRequestsTimeout > 0) { @@ -50,24 +61,33 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { } @Override - public ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames) { - UUID scriptId = UUID.randomUUID(); - String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_'); - String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames); - return doEval(scriptId, functionName, jsScript); + public ListenableFuture eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames) { + if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) { + UUID scriptId = UUID.randomUUID(); + String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_'); + String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames); + return doEval(scriptId, functionName, jsScript); + } else { + return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!")); + } } @Override - public ListenableFuture invokeFunction(UUID scriptId, Object... args) { - String functionName = scriptIdToNameMap.get(scriptId); - if (functionName == null) { - return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!")); - } - if (!isBlackListed(scriptId)) { - return doInvokeFunction(scriptId, functionName, args); + public ListenableFuture invokeFunction(TenantId tenantId, UUID scriptId, Object... args) { + if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) { + String functionName = scriptIdToNameMap.get(scriptId); + if (functionName == null) { + return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!")); + } + if (!isDisabled(scriptId)) { + apiUsageClient.report(tenantId, ApiUsageRecordKey.JS_EXEC_COUNT, 1); + return doInvokeFunction(scriptId, functionName, args); + } else { + return Futures.immediateFailedFuture( + new RuntimeException("Script invocation is blocked due to maximum error count " + getMaxErrors() + "!")); + } } else { - return Futures.immediateFailedFuture( - new RuntimeException("Script is blacklisted due to maximum error count " + getMaxErrors() + "!")); + return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!")); } } @@ -77,7 +97,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { if (functionName != null) { try { scriptIdToNameMap.remove(scriptId); - blackListedFunctions.remove(scriptId); + disabledFunctions.remove(scriptId); doRelease(scriptId, functionName); } catch (Exception e) { return Futures.immediateFailedFuture(e); @@ -97,7 +117,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract long getMaxBlacklistDuration(); protected void onScriptExecutionError(UUID scriptId) { - blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); + disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).incrementAndGet(); } private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { @@ -107,11 +127,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); } - private boolean isBlackListed(UUID scriptId) { - BlackListInfo errorCount = blackListedFunctions.get(scriptId); + private boolean isDisabled(UUID scriptId) { + DisableListInfo errorCount = disabledFunctions.get(scriptId); if (errorCount != null) { if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { - blackListedFunctions.remove(scriptId); + disabledFunctions.remove(scriptId); return false; } else { return errorCount.get() >= getMaxErrors(); @@ -121,11 +141,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { } } - private class BlackListInfo { + private class DisableListInfo { private final AtomicInteger counter; private long expirationTime; - private BlackListInfo() { + private DisableListInfo() { this.counter = new AtomicInteger(0); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index 8e6666bbba..db8bb68b4e 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -24,10 +24,10 @@ import delight.nashornsandbox.NashornSandboxes; import jdk.nashorn.api.scripting.NashornScriptEngineFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; -import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -38,7 +38,6 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -57,9 +56,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer private final FutureCallback evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); private final FutureCallback invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); - @Autowired @Getter - private JsExecutorService jsExecutor; + private final JsExecutorService jsExecutor; @Value("${js.local.max_requests_timeout:0}") private long maxRequestsTimeout; @@ -67,6 +65,11 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer @Value("${js.local.stats.enabled:false}") private boolean statsEnabled; + public AbstractNashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) { + super(apiUsageStateService, apiUsageClient); + this.jsExecutor = jsExecutor; + } + @Scheduled(fixedDelayString = "${js.local.stats.print_interval_ms:10000}") public void printStats() { if (statsEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java b/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java index 97e40e6f62..15fd8c7534 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java @@ -27,7 +27,7 @@ public class JsExecutorService extends AbstractListeningExecutor { @Override protected int getThreadPollSize() { - return jsExecutorThreadPoolSize; + return Math.max(jsExecutorThreadPoolSize, 1); } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java index a9f727e4a7..ae990af66c 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java @@ -17,14 +17,15 @@ package org.thingsboard.server.service.script; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; public interface JsInvokeService { - ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames); + ListenableFuture eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames); - ListenableFuture invokeFunction(UUID scriptId, Object... args); + ListenableFuture invokeFunction(TenantId tenantId, UUID scriptId, Object... args); ListenableFuture release(UUID scriptId); diff --git a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java index 66a14cc827..0dd45d340c 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java @@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import java.util.concurrent.TimeUnit; @@ -42,6 +44,10 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { @Value("${js.local.max_black_list_duration_sec:60}") private int maxBlackListDurationSec; + public NashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) { + super(apiUsageStateService, apiUsageClient, jsExecutor); + } + @Override protected boolean useJsSandbox() { return useJsSandbox; diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index cfb0c8a8ee..7b4114dce6 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -30,6 +30,8 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -68,6 +70,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { private final AtomicInteger queueFailedMsgs = new AtomicInteger(0); private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0); + public RemoteJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) { + super(apiUsageStateService, apiUsageClient); + } + @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}") public void printStats() { if (statsEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 110f410fad..275e645ce2 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -25,6 +25,7 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -43,13 +44,15 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S private final JsInvokeService sandboxService; private final UUID scriptId; + private final TenantId tenantId; private final EntityId entityId; - public RuleNodeJsScriptEngine(JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) { + public RuleNodeJsScriptEngine(TenantId tenantId, JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) { + this.tenantId = tenantId; this.sandboxService = sandboxService; this.entityId = entityId; try { - this.scriptId = this.sandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, script, argNames).get(); + this.scriptId = this.sandboxService.eval(tenantId, JsScriptType.RULE_NODE_SCRIPT, script, argNames).get(); } catch (Exception e) { Throwable t = e; if (e instanceof ExecutionException) { @@ -203,7 +206,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S private JsonNode executeScript(TbMsg msg) throws ScriptException { try { String[] inArgs = prepareArgs(msg); - String eval = sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString(); + String eval = sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString(); return mapper.readTree(eval); } catch (ExecutionException e) { if (e.getCause() instanceof ScriptException) { @@ -220,7 +223,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S private ListenableFuture executeScriptAsync(TbMsg msg) { String[] inArgs = prepareArgs(msg); - return Futures.transformAsync(sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]), + return Futures.transformAsync(sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]), o -> { try { return Futures.immediateFuture(mapper.readTree(o.toString())); diff --git a/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java b/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java index 91f741c308..47483fe147 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java +++ b/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java @@ -25,6 +25,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -34,6 +35,7 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; @@ -56,8 +58,10 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -118,6 +122,9 @@ public class AccessValidator { @Autowired protected AccessControlService accessControlService; + @Autowired + protected ApiUsageStateService apiUsageStateService; + private ExecutorService executor; @PostConstruct @@ -159,7 +166,11 @@ public class AccessValidator { new FutureCallback>() { @Override public void onSuccess(@Nullable DeferredResult result) { - onSuccess.accept(response, currentUser.getTenantId(), entityId); + try { + onSuccess.accept(response, currentUser.getTenantId(), entityId); + } catch (Exception e) { + onFailure(e); + } } @Override @@ -203,6 +214,9 @@ public class AccessValidator { case EDGE: validateEdge(currentUser, operation, entityId, callback); return; + case API_USAGE_STATE: + validateApiUsageState(currentUser, operation, entityId, callback); + return; default: //TODO: add support of other entities throw new IllegalStateException("Not Implemented!"); @@ -247,6 +261,27 @@ public class AccessValidator { } } + private void validateApiUsageState(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback callback) { + if (currentUser.isSystemAdmin()) { + callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION)); + } else { + if (!operation.equals(Operation.READ_TELEMETRY)) { + callback.onSuccess(ValidationResult.accessDenied("Allowed only READ_TELEMETRY operation!")); + } + ApiUsageState apiUsageState = apiUsageStateService.findApiUsageStateById(currentUser.getTenantId(), new ApiUsageStateId(entityId.getId())); + if (apiUsageState == null) { + callback.onSuccess(ValidationResult.entityNotFound("Api Usage State with requested id wasn't found!")); + } else { + try { + accessControlService.checkPermission(currentUser, Resource.API_USAGE_STATE, operation, entityId, apiUsageState); + } catch (ThingsboardException e) { + callback.onSuccess(ValidationResult.accessDenied(e.getMessage())); + } + callback.onSuccess(ValidationResult.ok(apiUsageState)); + } + } + } + private void validateAsset(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback callback) { if (currentUser.isSystemAdmin()) { callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION)); @@ -434,9 +469,9 @@ public class AccessValidator { public static void handleError(Throwable e, final DeferredResult response, HttpStatus defaultErrorStatus) { ResponseEntity responseEntity; - if (e != null && e instanceof ToErrorResponseEntity) { + if (e instanceof ToErrorResponseEntity) { responseEntity = ((ToErrorResponseEntity) e).toErrorResponseEntity(); - } else if (e != null && e instanceof IllegalArgumentException) { + } else if (e instanceof IllegalArgumentException || e instanceof IncorrectParameterException) { responseEntity = new ResponseEntity<>(e.getMessage(), HttpStatus.BAD_REQUEST); } else { responseEntity = new ResponseEntity<>(defaultErrorStatus); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java index c69721542f..ea6e7d4f19 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.security.auth.oauth2; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.base.Strings; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; @@ -32,24 +31,27 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.oauth2.OAuth2User; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.service.install.InstallScripts; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import java.io.IOException; import java.util.List; import java.util.Optional; -import java.util.concurrent.ExecutionException; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -77,14 +79,23 @@ public abstract class AbstractOAuth2ClientMapper { @Autowired private InstallScripts installScripts; + @Autowired + protected TbTenantProfileCache tenantProfileCache; + + @Autowired + protected TbClusterService tbClusterService; + private final Lock userCreationLock = new ReentrantLock(); - protected SecurityUser getOrCreateSecurityUserFromOAuth2User(OAuth2User oauth2User, boolean allowUserCreation, boolean activateUser) { + protected SecurityUser getOrCreateSecurityUserFromOAuth2User(OAuth2User oauth2User, OAuth2ClientRegistrationInfo clientRegistration) { + + OAuth2MapperConfig config = clientRegistration.getMapperConfig(); + UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, oauth2User.getEmail()); User user = userService.findUserByEmail(TenantId.SYS_TENANT_ID, oauth2User.getEmail()); - if (user == null && !allowUserCreation) { + if (user == null && !config.isAllowUserCreation()) { throw new UsernameNotFoundException("User not found: " + oauth2User.getEmail()); } @@ -109,21 +120,28 @@ public abstract class AbstractOAuth2ClientMapper { user.setFirstName(oauth2User.getFirstName()); user.setLastName(oauth2User.getLastName()); + ObjectNode additionalInfo = objectMapper.createObjectNode(); + if (!StringUtils.isEmpty(oauth2User.getDefaultDashboardName())) { Optional dashboardIdOpt = user.getAuthority() == Authority.TENANT_ADMIN ? getDashboardId(tenantId, oauth2User.getDefaultDashboardName()) : getDashboardId(tenantId, customerId, oauth2User.getDefaultDashboardName()); if (dashboardIdOpt.isPresent()) { - ObjectNode additionalInfo = objectMapper.createObjectNode(); additionalInfo.put("defaultDashboardFullscreen", oauth2User.isAlwaysFullScreen()); additionalInfo.put("defaultDashboardId", dashboardIdOpt.get().getId().toString()); - user.setAdditionalInfo(additionalInfo); } } + if (clientRegistration.getAdditionalInfo() != null && + clientRegistration.getAdditionalInfo().has("providerName")) { + additionalInfo.put("authProviderName", clientRegistration.getAdditionalInfo().get("providerName").asText()); + } + + user.setAdditionalInfo(additionalInfo); + user = userService.saveUser(user); - if (activateUser) { + if (config.isActivateUser()) { UserCredentials userCredentials = userService.findUserCredentialsByUserId(user.getTenantId(), user.getId()); userService.activateUserCredentials(user.getTenantId(), userCredentials.getActivateToken(), passwordEncoder.encode("")); } @@ -153,6 +171,10 @@ public abstract class AbstractOAuth2ClientMapper { tenant.setTitle(tenantName); tenant = tenantService.saveTenant(tenant); installScripts.createDefaultRuleChains(tenant.getId()); + tenantProfileCache.evict(tenant.getId()); + tbClusterService.onTenantChange(tenant, null); + tbClusterService.onEntityStateChange(tenant.getId(), tenant.getId(), + ComponentLifecycleEvent.CREATED); } else { tenant = tenants.get(0); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicMapperUtils.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicMapperUtils.java new file mode 100644 index 0000000000..246e74245d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicMapperUtils.java @@ -0,0 +1,78 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.text.StrSubstitutor; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig; +import org.thingsboard.server.dao.oauth2.OAuth2User; + +import java.util.Map; + +@Slf4j +public class BasicMapperUtils { + private static final String START_PLACEHOLDER_PREFIX = "%{"; + private static final String END_PLACEHOLDER_PREFIX = "}"; + + public static OAuth2User getOAuth2User(String email, Map attributes, OAuth2MapperConfig config) { + OAuth2User oauth2User = new OAuth2User(); + oauth2User.setEmail(email); + oauth2User.setTenantName(getTenantName(email, attributes, config)); + if (!StringUtils.isEmpty(config.getBasic().getLastNameAttributeKey())) { + String lastName = getStringAttributeByKey(attributes, config.getBasic().getLastNameAttributeKey()); + oauth2User.setLastName(lastName); + } + if (!StringUtils.isEmpty(config.getBasic().getFirstNameAttributeKey())) { + String firstName = getStringAttributeByKey(attributes, config.getBasic().getFirstNameAttributeKey()); + oauth2User.setFirstName(firstName); + } + if (!StringUtils.isEmpty(config.getBasic().getCustomerNamePattern())) { + StrSubstitutor sub = new StrSubstitutor(attributes, START_PLACEHOLDER_PREFIX, END_PLACEHOLDER_PREFIX); + String customerName = sub.replace(config.getBasic().getCustomerNamePattern()); + oauth2User.setCustomerName(customerName); + } + oauth2User.setAlwaysFullScreen(config.getBasic().isAlwaysFullScreen()); + if (!StringUtils.isEmpty(config.getBasic().getDefaultDashboardName())) { + oauth2User.setDefaultDashboardName(config.getBasic().getDefaultDashboardName()); + } + return oauth2User; + } + + public static String getTenantName(String email, Map attributes, OAuth2MapperConfig config) { + switch (config.getBasic().getTenantNameStrategy()) { + case EMAIL: + return email; + case DOMAIN: + return email.substring(email .indexOf("@") + 1); + case CUSTOM: + StrSubstitutor sub = new StrSubstitutor(attributes, START_PLACEHOLDER_PREFIX, END_PLACEHOLDER_PREFIX); + return sub.replace(config.getBasic().getTenantNamePattern()); + default: + throw new RuntimeException("Tenant Name Strategy with type " + config.getBasic().getTenantNameStrategy() + " is not supported!"); + } + } + + public static String getStringAttributeByKey(Map attributes, String key) { + String result = null; + try { + result = (String) attributes.get(key); + } catch (Exception e) { + log.warn("Can't convert attribute to String by key " + key); + } + return result; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java index 2170479f92..bd6434f50a 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java @@ -16,11 +16,10 @@ package org.thingsboard.server.service.security.auth.oauth2; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.text.StrSubstitutor; import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; -import org.thingsboard.server.dao.oauth2.OAuth2ClientMapperConfig; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig; import org.thingsboard.server.dao.oauth2.OAuth2User; import org.thingsboard.server.service.security.model.SecurityUser; @@ -30,62 +29,13 @@ import java.util.Map; @Slf4j public class BasicOAuth2ClientMapper extends AbstractOAuth2ClientMapper implements OAuth2ClientMapper { - private static final String START_PLACEHOLDER_PREFIX = "%{"; - private static final String END_PLACEHOLDER_PREFIX = "}"; - private static final String EMAIL_TENANT_STRATEGY = "email"; - private static final String DOMAIN_TENANT_STRATEGY = "domain"; - private static final String CUSTOM_TENANT_STRATEGY = "custom"; - @Override - public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig config) { - OAuth2User oauth2User = new OAuth2User(); + public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, String providerAccessToken, OAuth2ClientRegistrationInfo clientRegistration) { + OAuth2MapperConfig config = clientRegistration.getMapperConfig(); Map attributes = token.getPrincipal().getAttributes(); - String email = getStringAttributeByKey(attributes, config.getBasic().getEmailAttributeKey()); - oauth2User.setEmail(email); - oauth2User.setTenantName(getTenantName(attributes, config)); - if (!StringUtils.isEmpty(config.getBasic().getLastNameAttributeKey())) { - String lastName = getStringAttributeByKey(attributes, config.getBasic().getLastNameAttributeKey()); - oauth2User.setLastName(lastName); - } - if (!StringUtils.isEmpty(config.getBasic().getFirstNameAttributeKey())) { - String firstName = getStringAttributeByKey(attributes, config.getBasic().getFirstNameAttributeKey()); - oauth2User.setFirstName(firstName); - } - if (!StringUtils.isEmpty(config.getBasic().getCustomerNamePattern())) { - StrSubstitutor sub = new StrSubstitutor(attributes, START_PLACEHOLDER_PREFIX, END_PLACEHOLDER_PREFIX); - String customerName = sub.replace(config.getBasic().getCustomerNamePattern()); - oauth2User.setCustomerName(customerName); - } - oauth2User.setAlwaysFullScreen(config.getBasic().isAlwaysFullScreen()); - if (!StringUtils.isEmpty(config.getBasic().getDefaultDashboardName())) { - oauth2User.setDefaultDashboardName(config.getBasic().getDefaultDashboardName()); - } - - return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.isAllowUserCreation(), config.isActivateUser()); - } - - private String getTenantName(Map attributes, OAuth2ClientMapperConfig config) { - switch (config.getBasic().getTenantNameStrategy()) { - case EMAIL_TENANT_STRATEGY: - return getStringAttributeByKey(attributes, config.getBasic().getEmailAttributeKey()); - case DOMAIN_TENANT_STRATEGY: - String email = getStringAttributeByKey(attributes, config.getBasic().getEmailAttributeKey()); - return email.substring(email .indexOf("@") + 1); - case CUSTOM_TENANT_STRATEGY: - StrSubstitutor sub = new StrSubstitutor(attributes, START_PLACEHOLDER_PREFIX, END_PLACEHOLDER_PREFIX); - return sub.replace(config.getBasic().getTenantNamePattern()); - default: - throw new RuntimeException("Tenant Name Strategy with type " + config.getBasic().getTenantNameStrategy() + " is not supported!"); - } - } + String email = BasicMapperUtils.getStringAttributeByKey(attributes, config.getBasic().getEmailAttributeKey()); + OAuth2User oauth2User = BasicMapperUtils.getOAuth2User(email, attributes, config); - private String getStringAttributeByKey(Map attributes, String key) { - String result = null; - try { - result = (String) attributes.get(key); - } catch (Exception e) { - log.warn("Can't convert attribute to String by key " + key); - } - return result; + return getOrCreateSecurityUserFromOAuth2User(oauth2User, clientRegistration); } } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java index 42ba95b4ee..0cd292f76f 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java @@ -23,28 +23,36 @@ import org.springframework.security.oauth2.client.authentication.OAuth2Authentic import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.springframework.web.client.RestTemplate; -import org.thingsboard.server.dao.oauth2.OAuth2ClientMapperConfig; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2CustomMapperConfig; +import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig; import org.thingsboard.server.dao.oauth2.OAuth2User; import org.thingsboard.server.service.security.model.SecurityUser; @Service(value = "customOAuth2ClientMapper") @Slf4j public class CustomOAuth2ClientMapper extends AbstractOAuth2ClientMapper implements OAuth2ClientMapper { + private static final String PROVIDER_ACCESS_TOKEN = "provider-access-token"; private static final ObjectMapper json = new ObjectMapper(); private RestTemplateBuilder restTemplateBuilder = new RestTemplateBuilder(); @Override - public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig config) { - OAuth2User oauth2User = getOAuth2User(token, config.getCustom()); - return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.isAllowUserCreation(), config.isActivateUser()); + public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, String providerAccessToken, OAuth2ClientRegistrationInfo clientRegistration) { + OAuth2MapperConfig config = clientRegistration.getMapperConfig(); + OAuth2User oauth2User = getOAuth2User(token, providerAccessToken, config.getCustom()); + return getOrCreateSecurityUserFromOAuth2User(oauth2User, clientRegistration); } - private synchronized OAuth2User getOAuth2User(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig.CustomOAuth2ClientMapperConfig custom) { + private synchronized OAuth2User getOAuth2User(OAuth2AuthenticationToken token, String providerAccessToken, OAuth2CustomMapperConfig custom) { if (!StringUtils.isEmpty(custom.getUsername()) && !StringUtils.isEmpty(custom.getPassword())) { restTemplateBuilder = restTemplateBuilder.basicAuthentication(custom.getUsername(), custom.getPassword()); } + if (custom.isSendToken() && !StringUtils.isEmpty(providerAccessToken)) { + restTemplateBuilder = restTemplateBuilder.defaultHeader(PROVIDER_ACCESS_TOKEN, providerAccessToken); + } + RestTemplate restTemplate = restTemplateBuilder.build(); String request; try { diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/GithubOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/GithubOAuth2ClientMapper.java new file mode 100644 index 0000000000..f259720626 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/GithubOAuth2ClientMapper.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import lombok.Data; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig; +import org.thingsboard.server.dao.oauth2.OAuth2Configuration; +import org.thingsboard.server.dao.oauth2.OAuth2User; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.ArrayList; +import java.util.Map; +import java.util.Optional; + +@Service(value = "githubOAuth2ClientMapper") +@Slf4j +public class GithubOAuth2ClientMapper extends AbstractOAuth2ClientMapper implements OAuth2ClientMapper { + private static final String EMAIL_URL_KEY = "emailUrl"; + + private static final String AUTHORIZATION = "Authorization"; + + private RestTemplateBuilder restTemplateBuilder = new RestTemplateBuilder(); + + @Autowired + private OAuth2Configuration oAuth2Configuration; + + @Override + public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, String providerAccessToken, OAuth2ClientRegistrationInfo clientRegistration) { + OAuth2MapperConfig config = clientRegistration.getMapperConfig(); + Map githubMapperConfig = oAuth2Configuration.getGithubMapper(); + String email = getEmail(githubMapperConfig.get(EMAIL_URL_KEY), providerAccessToken); + Map attributes = token.getPrincipal().getAttributes(); + OAuth2User oAuth2User = BasicMapperUtils.getOAuth2User(email, attributes, config); + return getOrCreateSecurityUserFromOAuth2User(oAuth2User, clientRegistration); + } + + private synchronized String getEmail(String emailUrl, String oauth2Token) { + restTemplateBuilder = restTemplateBuilder.defaultHeader(AUTHORIZATION, "token " + oauth2Token); + + RestTemplate restTemplate = restTemplateBuilder.build(); + GithubEmailsResponse githubEmailsResponse; + try { + githubEmailsResponse = restTemplate.getForEntity(emailUrl, GithubEmailsResponse.class).getBody(); + if (githubEmailsResponse == null){ + throw new RuntimeException("Empty Github response!"); + } + } catch (Exception e) { + log.error("There was an error during connection to Github API", e); + throw new RuntimeException("Unable to login. Please contact your Administrator!"); + } + Optional emailOpt = githubEmailsResponse.stream() + .filter(GithubEmailResponse::isPrimary) + .map(GithubEmailResponse::getEmail) + .findAny(); + if (emailOpt.isPresent()){ + return emailOpt.get(); + } else { + log.error("Could not find primary email from {}.", githubEmailsResponse); + throw new RuntimeException("Unable to login. Please contact your Administrator!"); + } + } + private static class GithubEmailsResponse extends ArrayList {} + + @Data + @ToString + private static class GithubEmailResponse { + private String email; + private boolean verified; + private boolean primary; + private String visibility; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java index 196bfe7b50..4c9804930e 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java @@ -16,9 +16,9 @@ package org.thingsboard.server.service.security.auth.oauth2; import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; -import org.thingsboard.server.dao.oauth2.OAuth2ClientMapperConfig; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo; import org.thingsboard.server.service.security.model.SecurityUser; public interface OAuth2ClientMapper { - SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig config); + SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, String providerAccessToken, OAuth2ClientRegistrationInfo clientRegistration); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java index e1c5b694bb..2e764fe223 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.oauth2.MapperType; @Component @Slf4j @@ -32,14 +33,20 @@ public class OAuth2ClientMapperProvider { @Qualifier("customOAuth2ClientMapper") private OAuth2ClientMapper customOAuth2ClientMapper; - public OAuth2ClientMapper getOAuth2ClientMapperByType(String oauth2ClientType) { - switch (oauth2ClientType) { - case "custom": + @Autowired + @Qualifier("githubOAuth2ClientMapper") + private OAuth2ClientMapper githubOAuth2ClientMapper; + + public OAuth2ClientMapper getOAuth2ClientMapperByType(MapperType oauth2MapperType) { + switch (oauth2MapperType) { + case CUSTOM: return customOAuth2ClientMapper; - case "basic": + case BASIC: return basicOAuth2ClientMapper; + case GITHUB: + return githubOAuth2ClientMapper; default: - throw new RuntimeException("OAuth2ClientMapper with type " + oauth2ClientType + " is not supported!"); + throw new RuntimeException("OAuth2ClientRegistrationMapper with type " + oauth2MapperType + " is not supported!"); } } } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java index 653be85f72..27a4962a0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java @@ -15,10 +15,15 @@ */ package org.thingsboard.server.service.security.auth.oauth2; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.security.core.AuthenticationException; import org.springframework.security.web.authentication.SimpleUrlAuthenticationFailureHandler; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.utils.MiscUtils; import javax.servlet.ServletException; @@ -30,13 +35,20 @@ import java.nio.charset.StandardCharsets; @Component(value = "oauth2AuthenticationFailureHandler") @ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") -public class Oauth2AuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler { +public class Oauth2AuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler { + + private final SystemSecurityService systemSecurityService; + + @Autowired + public Oauth2AuthenticationFailureHandler(final SystemSecurityService systemSecurityService) { + this.systemSecurityService = systemSecurityService; + } @Override public void onAuthenticationFailure(HttpServletRequest request, HttpServletResponse response, AuthenticationException exception) throws IOException, ServletException { - String baseUrl = MiscUtils.constructBaseUrl(request); + String baseUrl = this.systemSecurityService.getBaseUrl(TenantId.SYS_TENANT_ID, new CustomerId(EntityId.NULL_UUID), request); getRedirectStrategy().sendRedirect(request, response, baseUrl + "/login?loginError=" + URLEncoder.encode(exception.getMessage(), StandardCharsets.UTF_8.toString())); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java index 3375d046e2..2e2eeeee61 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java @@ -16,17 +16,22 @@ package org.thingsboard.server.service.security.auth.oauth2; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.security.core.Authentication; +import org.springframework.security.oauth2.client.OAuth2AuthorizedClient; +import org.springframework.security.oauth2.client.OAuth2AuthorizedClientService; import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; import org.springframework.security.web.authentication.SimpleUrlAuthenticationSuccessHandler; import org.springframework.stereotype.Component; -import org.thingsboard.server.dao.oauth2.OAuth2Client; -import org.thingsboard.server.dao.oauth2.OAuth2Configuration; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo; +import org.thingsboard.server.dao.oauth2.OAuth2Service; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtToken; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; +import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.utils.MiscUtils; import javax.servlet.http.HttpServletRequest; @@ -34,39 +39,47 @@ import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.util.UUID; @Component(value = "oauth2AuthenticationSuccessHandler") -@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationSuccessHandler { private final JwtTokenFactory tokenFactory; private final RefreshTokenRepository refreshTokenRepository; private final OAuth2ClientMapperProvider oauth2ClientMapperProvider; - private final OAuth2Configuration oauth2Configuration; + private final OAuth2Service oAuth2Service; + private final OAuth2AuthorizedClientService oAuth2AuthorizedClientService; + private final SystemSecurityService systemSecurityService; @Autowired public Oauth2AuthenticationSuccessHandler(final JwtTokenFactory tokenFactory, final RefreshTokenRepository refreshTokenRepository, final OAuth2ClientMapperProvider oauth2ClientMapperProvider, - final OAuth2Configuration oauth2Configuration) { + final OAuth2Service oAuth2Service, + final OAuth2AuthorizedClientService oAuth2AuthorizedClientService, final SystemSecurityService systemSecurityService) { this.tokenFactory = tokenFactory; this.refreshTokenRepository = refreshTokenRepository; this.oauth2ClientMapperProvider = oauth2ClientMapperProvider; - this.oauth2Configuration = oauth2Configuration; + this.oAuth2Service = oAuth2Service; + this.oAuth2AuthorizedClientService = oAuth2AuthorizedClientService; + this.systemSecurityService = systemSecurityService; } @Override public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws IOException { - - String baseUrl = MiscUtils.constructBaseUrl(request); + String baseUrl = this.systemSecurityService.getBaseUrl(TenantId.SYS_TENANT_ID, new CustomerId(EntityId.NULL_UUID), request); try { OAuth2AuthenticationToken token = (OAuth2AuthenticationToken) authentication; - OAuth2Client oauth2Client = oauth2Configuration.getClientByRegistrationId(token.getAuthorizedClientRegistrationId()); - OAuth2ClientMapper mapper = oauth2ClientMapperProvider.getOAuth2ClientMapperByType(oauth2Client.getMapperConfig().getType()); - SecurityUser securityUser = mapper.getOrCreateUserByClientPrincipal(token, oauth2Client.getMapperConfig()); + OAuth2ClientRegistrationInfo clientRegistration = oAuth2Service.findClientRegistrationInfo(UUID.fromString(token.getAuthorizedClientRegistrationId())); + OAuth2AuthorizedClient oAuth2AuthorizedClient = oAuth2AuthorizedClientService.loadAuthorizedClient( + token.getAuthorizedClientRegistrationId(), + token.getPrincipal().getName()); + OAuth2ClientMapper mapper = oauth2ClientMapperProvider.getOAuth2ClientMapperByType(clientRegistration.getMapperConfig().getType()); + SecurityUser securityUser = mapper.getOrCreateUserByClientPrincipal(token, oAuth2AuthorizedClient.getAccessToken().getTokenValue(), + clientRegistration); JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser); JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser); @@ -77,4 +90,4 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS URLEncoder.encode(e.getMessage(), StandardCharsets.UTF_8.toString())); } } -} \ No newline at end of file +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 3d3273257a..0ee1fa9266 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -33,8 +33,11 @@ public enum Resource { USER(EntityType.USER), WIDGETS_BUNDLE(EntityType.WIDGETS_BUNDLE), WIDGET_TYPE(EntityType.WIDGET_TYPE), + OAUTH2_CONFIGURATION_INFO(), + OAUTH2_CONFIGURATION_TEMPLATE(), TENANT_PROFILE(EntityType.TENANT_PROFILE), - DEVICE_PROFILE(EntityType.DEVICE_PROFILE); + DEVICE_PROFILE(EntityType.DEVICE_PROFILE), + API_USAGE_STATE(EntityType.API_USAGE_STATE); private final EntityType entityType; diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java index 766290298a..abed9d5bc7 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java @@ -19,14 +19,10 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.model.SecurityUser; -import java.util.HashMap; -import java.util.Optional; - @Component(value="sysAdminPermissions") public class SysAdminPermissions extends AbstractPermissions { @@ -39,6 +35,8 @@ public class SysAdminPermissions extends AbstractPermissions { put(Resource.USER, userPermissionChecker); put(Resource.WIDGETS_BUNDLE, systemEntityPermissionChecker); put(Resource.WIDGET_TYPE, systemEntityPermissionChecker); + put(Resource.OAUTH2_CONFIGURATION_INFO, PermissionChecker.allowAllPermissionChecker); + put(Resource.OAUTH2_CONFIGURATION_TEMPLATE, PermissionChecker.allowAllPermissionChecker); put(Resource.TENANT_PROFILE, PermissionChecker.allowAllPermissionChecker); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index a577c3158a..ade92c6572 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -19,13 +19,10 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.model.SecurityUser; -import java.util.HashMap; - @Component(value="tenantAdminPermissions") public class TenantAdminPermissions extends AbstractPermissions { @@ -43,6 +40,7 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); put(Resource.WIDGET_TYPE, widgetsPermissionChecker); put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); + put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker); put(Resource.EDGE, tenantEntityPermissionChecker); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java index 11857f25b6..9bd22c88f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java @@ -40,17 +40,20 @@ import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.UserCredentials; +import org.thingsboard.server.common.data.security.model.SecuritySettings; +import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserServiceImpl; import org.thingsboard.server.service.security.exception.UserPasswordExpiredException; -import org.thingsboard.server.common.data.security.model.SecuritySettings; -import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; +import org.thingsboard.server.utils.MiscUtils; import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -146,7 +149,7 @@ public class DefaultSystemSecurityService implements SystemSecurityService { if (isPositiveInteger(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays())) { if ((userCredentials.getCreatedTime() + TimeUnit.DAYS.toMillis(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays())) - < System.currentTimeMillis()) { + < System.currentTimeMillis()) { userCredentials = userService.requestExpiredPasswordReset(tenantId, userCredentials.getId()); throw new UserPasswordExpiredException("User password expired!", userCredentials.getResetToken()); } @@ -197,6 +200,24 @@ public class DefaultSystemSecurityService implements SystemSecurityService { } } + @Override + public String getBaseUrl(TenantId tenantId, CustomerId customerId, HttpServletRequest httpServletRequest) { + String baseUrl = null; + AdminSettings generalSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "general"); + + JsonNode prohibitDifferentUrl = generalSettings.getJsonValue().get("prohibitDifferentUrl"); + + if (prohibitDifferentUrl != null && prohibitDifferentUrl.asBoolean()) { + baseUrl = generalSettings.getJsonValue().get("baseUrl").asText(); + } + + if (StringUtils.isEmpty(baseUrl)) { + baseUrl = MiscUtils.constructBaseUrl(httpServletRequest); + } + + return baseUrl; + } + private static boolean isPositiveInteger(Integer val) { return val != null && val.intValue() > 0; } diff --git a/application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java b/application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java index afb7834cef..01367d53f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java @@ -16,11 +16,14 @@ package org.thingsboard.server.service.security.system; import org.springframework.security.core.AuthenticationException; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.common.data.security.model.SecuritySettings; +import javax.servlet.http.HttpServletRequest; + public interface SystemSecurityService { SecuritySettings getSecuritySettings(TenantId tenantId); @@ -31,4 +34,6 @@ public interface SystemSecurityService { void validatePassword(TenantId tenantId, String password, UserCredentials userCredentials) throws DataValidationException; + String getBaseUrl(TenantId tenantId, CustomerId customerId, HttpServletRequest httpServletRequest); + } diff --git a/application/src/main/java/org/thingsboard/server/service/sms/AbstractSmsSender.java b/application/src/main/java/org/thingsboard/server/service/sms/AbstractSmsSender.java new file mode 100644 index 0000000000..98c96c3564 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sms/AbstractSmsSender.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sms; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.sms.SmsSender; +import org.thingsboard.rule.engine.api.sms.exception.SmsParseException; + +import java.util.regex.Pattern; + +@Slf4j +public abstract class AbstractSmsSender implements SmsSender { + + private static final Pattern E_164_PHONE_NUMBER_PATTERN = Pattern.compile("^\\+[1-9]\\d{1,14}$"); + + private static final int MAX_SMS_MESSAGE_LENGTH = 1600; + private static final int MAX_SMS_SEGMENT_LENGTH = 70; + + protected String validatePhoneNumber(String phoneNumber) throws SmsParseException { + phoneNumber = phoneNumber.trim(); + if (!E_164_PHONE_NUMBER_PATTERN.matcher(phoneNumber).matches()) { + throw new SmsParseException("Invalid phone number format. Phone number must be in E.164 format."); + } + return phoneNumber; + } + + protected String prepareMessage(String message) { + message = message.replaceAll("^\"|\"$", "").replaceAll("\\\\n", "\n"); + if (message.length() > MAX_SMS_MESSAGE_LENGTH) { + log.warn("SMS message exceeds maximum symbols length and will be truncated"); + message = message.substring(0, MAX_SMS_MESSAGE_LENGTH); + } + return message; + } + + protected int countMessageSegments(String message) { + return (int)Math.ceil((double) message.length() / (double) MAX_SMS_SEGMENT_LENGTH); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sms/DefaultSmsSenderFactory.java b/application/src/main/java/org/thingsboard/server/service/sms/DefaultSmsSenderFactory.java new file mode 100644 index 0000000000..d319276c40 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sms/DefaultSmsSenderFactory.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sms; + +import org.springframework.stereotype.Component; +import org.thingsboard.rule.engine.api.sms.SmsSender; +import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; +import org.thingsboard.server.common.data.sms.config.AwsSnsSmsProviderConfiguration; +import org.thingsboard.server.common.data.sms.config.SmsProviderConfiguration; +import org.thingsboard.server.common.data.sms.config.TwilioSmsProviderConfiguration; +import org.thingsboard.server.service.sms.aws.AwsSmsSender; +import org.thingsboard.server.service.sms.twilio.TwilioSmsSender; + +@Component +public class DefaultSmsSenderFactory implements SmsSenderFactory { + + @Override + public SmsSender createSmsSender(SmsProviderConfiguration config) { + switch (config.getType()) { + case AWS_SNS: + return new AwsSmsSender((AwsSnsSmsProviderConfiguration)config); + case TWILIO: + return new TwilioSmsSender((TwilioSmsProviderConfiguration)config); + default: + throw new RuntimeException("Unknown SMS provider type " + config.getType()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sms/DefaultSmsService.java b/application/src/main/java/org/thingsboard/server/service/sms/DefaultSmsService.java new file mode 100644 index 0000000000..2a8b2ef913 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sms/DefaultSmsService.java @@ -0,0 +1,145 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sms; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.NestedRuntimeException; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.SmsService; +import org.thingsboard.rule.engine.api.sms.SmsSender; +import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; +import org.thingsboard.server.common.data.sms.config.SmsProviderConfiguration; +import org.thingsboard.server.common.data.sms.config.TestSmsRequest; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.dao.util.mapping.JacksonUtil; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +@Service +@Slf4j +public class DefaultSmsService implements SmsService { + + private final SmsSenderFactory smsSenderFactory; + private final AdminSettingsService adminSettingsService; + private final TbApiUsageStateService apiUsageStateService; + private final TbApiUsageClient apiUsageClient; + + private SmsSender smsSender; + + public DefaultSmsService(SmsSenderFactory smsSenderFactory, AdminSettingsService adminSettingsService, TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) { + this.smsSenderFactory = smsSenderFactory; + this.adminSettingsService = adminSettingsService; + this.apiUsageStateService = apiUsageStateService; + this.apiUsageClient = apiUsageClient; + } + + @PostConstruct + private void init() { + updateSmsConfiguration(); + } + + @PreDestroy + private void destroy() { + if (this.smsSender != null) { + this.smsSender.destroy(); + } + } + + @Override + public void updateSmsConfiguration() { + AdminSettings settings = adminSettingsService.findAdminSettingsByKey(new TenantId(EntityId.NULL_UUID), "sms"); + if (settings != null) { + try { + JsonNode jsonConfig = settings.getJsonValue(); + SmsProviderConfiguration configuration = JacksonUtil.convertValue(jsonConfig, SmsProviderConfiguration.class); + SmsSender newSmsSender = this.smsSenderFactory.createSmsSender(configuration); + if (this.smsSender != null) { + this.smsSender.destroy(); + } + this.smsSender = newSmsSender; + } catch (Exception e) { + log.error("Failed to create SMS sender", e); + } + } + } + + private int sendSms(String numberTo, String message) throws ThingsboardException { + if (this.smsSender == null) { + throw new ThingsboardException("Unable to send SMS: no SMS provider configured!", ThingsboardErrorCode.GENERAL); + } + return this.sendSms(this.smsSender, numberTo, message); + } + + @Override + public void sendSms(TenantId tenantId, String[] numbersTo, String message) throws ThingsboardException { + if (apiUsageStateService.getApiUsageState(tenantId).isSmsSendEnabled()) { + int smsCount = 0; + try { + for (String numberTo : numbersTo) { + smsCount += this.sendSms(numberTo, message); + } + } finally { + if (smsCount > 0) { + apiUsageClient.report(tenantId, ApiUsageRecordKey.SMS_EXEC_COUNT, smsCount); + } + } + } else { + throw new RuntimeException("SMS sending is disabled due to API limits!"); + } + } + + @Override + public void sendTestSms(TestSmsRequest testSmsRequest) throws ThingsboardException { + SmsSender testSmsSender; + try { + testSmsSender = this.smsSenderFactory.createSmsSender(testSmsRequest.getProviderConfiguration()); + } catch (Exception e) { + throw handleException(e); + } + this.sendSms(testSmsSender, testSmsRequest.getNumberTo(), testSmsRequest.getMessage()); + testSmsSender.destroy(); + } + + private int sendSms(SmsSender smsSender, String numberTo, String message) throws ThingsboardException { + try { + return smsSender.sendSms(numberTo, message); + } catch (Exception e) { + throw handleException(e); + } + } + + private ThingsboardException handleException(Exception exception) { + String message; + if (exception instanceof NestedRuntimeException) { + message = ((NestedRuntimeException) exception).getMostSpecificCause().getMessage(); + } else { + message = exception.getMessage(); + } + log.warn("Unable to send SMS: {}", message); + return new ThingsboardException(String.format("Unable to send SMS: %s", message), + ThingsboardErrorCode.GENERAL); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sms/SmsExecutorService.java b/application/src/main/java/org/thingsboard/server/service/sms/SmsExecutorService.java new file mode 100644 index 0000000000..48aa2d5dc1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sms/SmsExecutorService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sms; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.AbstractListeningExecutor; + +@Component +public class SmsExecutorService extends AbstractListeningExecutor { + + @Value("${actors.rule.sms_thread_pool_size}") + private int smsExecutorThreadPoolSize; + + @Override + protected int getThreadPollSize() { + return smsExecutorThreadPoolSize; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sms/aws/AwsSmsSender.java b/application/src/main/java/org/thingsboard/server/service/sms/aws/AwsSmsSender.java new file mode 100644 index 0000000000..c6cea69540 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sms/aws/AwsSmsSender.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sms.aws; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sns.AmazonSNS; +import com.amazonaws.services.sns.AmazonSNSClient; +import com.amazonaws.services.sns.model.PublishRequest; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.thingsboard.server.common.data.sms.config.AwsSnsSmsProviderConfiguration; +import org.thingsboard.rule.engine.api.sms.exception.SmsException; +import org.thingsboard.rule.engine.api.sms.exception.SmsSendException; +import org.thingsboard.server.service.sms.AbstractSmsSender; + +@Slf4j +public class AwsSmsSender extends AbstractSmsSender { + + private AmazonSNS snsClient; + + public AwsSmsSender(AwsSnsSmsProviderConfiguration config) { + if (StringUtils.isEmpty(config.getAccessKeyId()) || StringUtils.isEmpty(config.getSecretAccessKey()) || StringUtils.isEmpty(config.getRegion())) { + throw new IllegalArgumentException("Invalid AWS sms provider configuration: aws accessKeyId, aws secretAccessKey and aws region should be specified!"); + } + AWSCredentials awsCredentials = new BasicAWSCredentials(config.getAccessKeyId(), config.getSecretAccessKey()); + AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); + this.snsClient = AmazonSNSClient.builder() + .withCredentials(credProvider) + .withRegion(config.getRegion()) + .build(); + } + + @Override + public int sendSms(String numberTo, String message) throws SmsException { + numberTo = this.validatePhoneNumber(numberTo); + message = this.prepareMessage(message); + try { + PublishRequest publishRequest = new PublishRequest() + .withPhoneNumber(numberTo) + .withMessage(message); + this.snsClient.publish(publishRequest); + return this.countMessageSegments(message); + } catch (Exception e) { + throw new SmsSendException("Failed to send SMS message - " + e.getMessage(), e); + } + } + + @Override + public void destroy() { + if (this.snsClient != null) { + try { + this.snsClient.shutdown(); + } catch (Exception e) { + log.error("Failed to shutdown SNS client during destroy()", e); + } + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sms/twilio/TwilioSmsSender.java b/application/src/main/java/org/thingsboard/server/service/sms/twilio/TwilioSmsSender.java new file mode 100644 index 0000000000..7af65cfc48 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sms/twilio/TwilioSmsSender.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sms.twilio; + +import com.twilio.http.TwilioRestClient; +import com.twilio.rest.api.v2010.account.Message; +import com.twilio.type.PhoneNumber; +import org.apache.commons.lang3.StringUtils; +import org.thingsboard.server.common.data.sms.config.TwilioSmsProviderConfiguration; +import org.thingsboard.rule.engine.api.sms.exception.SmsException; +import org.thingsboard.rule.engine.api.sms.exception.SmsSendException; +import org.thingsboard.server.service.sms.AbstractSmsSender; + +public class TwilioSmsSender extends AbstractSmsSender { + + private TwilioRestClient twilioRestClient; + private String numberFrom; + + public TwilioSmsSender(TwilioSmsProviderConfiguration config) { + if (StringUtils.isEmpty(config.getAccountSid()) || StringUtils.isEmpty(config.getAccountToken()) || StringUtils.isEmpty(config.getNumberFrom())) { + throw new IllegalArgumentException("Invalid twilio sms provider configuration: accountSid, accountToken and numberFrom should be specified!"); + } + this.numberFrom = this.validatePhoneNumber(config.getNumberFrom()); + this.twilioRestClient = new TwilioRestClient.Builder(config.getAccountSid(), config.getAccountToken()).build(); + } + + @Override + public int sendSms(String numberTo, String message) throws SmsException { + numberTo = this.validatePhoneNumber(numberTo); + message = this.prepareMessage(message); + try { + String numSegments = Message.creator(new PhoneNumber(numberTo), new PhoneNumber(this.numberFrom), message).create(this.twilioRestClient).getNumSegments(); + return Integer.valueOf(numSegments); + } catch (Exception e) { + throw new SmsSendException("Failed to send SMS message - " + e.getMessage(), e); + } + } + + @Override + public void destroy() { + + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 0772dd907b..234b64f94e 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -392,7 +392,7 @@ public class DefaultDeviceStateService implements DeviceStateService { if (stateData != null) { DeviceState state = stateData.getState(); state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); - if (!state.isActive() && (state.getLastInactivityAlarmTime() == 0L || state.getLastInactivityAlarmTime() < state.getLastActivityTime())) { + if (!state.isActive() && (state.getLastInactivityAlarmTime() == 0L || state.getLastInactivityAlarmTime() < state.getLastActivityTime()) && stateData.getDeviceCreationTime() + state.getInactivityTimeout() < ts) { state.setLastInactivityAlarmTime(ts); pushRuleEngineMessage(stateData, INACTIVITY_EVENT); save(deviceId, INACTIVITY_ALARM_TIME, ts); @@ -479,6 +479,7 @@ public class DefaultDeviceStateService implements DeviceStateService { return DeviceStateData.builder() .tenantId(device.getTenantId()) .deviceId(device.getId()) + .deviceCreationTime(device.getCreatedTime()) .metaData(md) .state(deviceState).build(); } catch (Exception e) { @@ -520,27 +521,27 @@ public class DefaultDeviceStateService implements DeviceStateService { private void save(DeviceId deviceId, String key, long value) { if (persistToTelemetry) { - tsSubService.saveAndNotify( + tsSubService.saveAndNotifyInternal( TenantId.SYS_TENANT_ID, deviceId, Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new LongDataEntry(key, value))), - new AttributeSaveCallback(deviceId, key, value)); + new AttributeSaveCallback<>(deviceId, key, value)); } else { - tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, DataConstants.SERVER_SCOPE, key, value, new AttributeSaveCallback(deviceId, key, value)); + tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, DataConstants.SERVER_SCOPE, key, value, new AttributeSaveCallback<>(deviceId, key, value)); } } private void save(DeviceId deviceId, String key, boolean value) { if (persistToTelemetry) { - tsSubService.saveAndNotify( + tsSubService.saveAndNotifyInternal( TenantId.SYS_TENANT_ID, deviceId, Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), - new AttributeSaveCallback(deviceId, key, value)); + new AttributeSaveCallback<>(deviceId, key, value)); } else { - tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, DataConstants.SERVER_SCOPE, key, value, new AttributeSaveCallback(deviceId, key, value)); + tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, DataConstants.SERVER_SCOPE, key, value, new AttributeSaveCallback<>(deviceId, key, value)); } } - private static class AttributeSaveCallback implements FutureCallback { + private static class AttributeSaveCallback implements FutureCallback { private final DeviceId deviceId; private final String key; private final Object value; @@ -552,7 +553,7 @@ public class DefaultDeviceStateService implements DeviceStateService { } @Override - public void onSuccess(@Nullable Void result) { + public void onSuccess(@Nullable T result) { log.trace("[{}] Successfully updated attribute [{}] with value [{}]", deviceId, key, value); } diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateData.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateData.java index 1973570f9e..3ace93b89a 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateData.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateData.java @@ -30,8 +30,8 @@ class DeviceStateData { private final TenantId tenantId; private final DeviceId deviceId; - + private final long deviceCreationTime; private TbMsgMetaData metaData; private final DeviceState state; - + } diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index b46033389b..a832eb0870 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -48,9 +48,9 @@ import java.util.stream.Collectors; public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; - public static final FutureCallback CALLBACK = new FutureCallback() { + public static final FutureCallback CALLBACK = new FutureCallback() { @Override - public void onSuccess(@Nullable Void result) { + public void onSuccess(@Nullable Integer result) { } @@ -85,7 +85,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) .collect(Collectors.toList()); if (!tsList.isEmpty()) { - tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + tsService.saveAndNotifyInternal(tenantId, serviceAssetId, tsList, CALLBACK); } } } catch (DataValidationException e) { @@ -97,7 +97,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); try { - tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); } catch (DataValidationException e2) { if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { throw e2; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 89233d2f4f..cd85c44cbe 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -21,12 +21,6 @@ import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; @@ -36,7 +30,6 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.telemetry.DefaultTelemetryWebSocketService; import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate; import org.thingsboard.server.service.telemetry.sub.TelemetrySubscriptionUpdate; @@ -49,7 +42,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.stream.Collectors; @Slf4j @TbCoreComponent @@ -59,9 +51,6 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer private final Set currentPartitions = ConcurrentHashMap.newKeySet(); private final Map> subscriptionsBySessionId = new ConcurrentHashMap<>(); - @Autowired - private EntityViewService entityViewService; - @Autowired private PartitionService partitionService; @@ -72,17 +61,17 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer @Lazy private SubscriptionManagerService subscriptionManagerService; - private ExecutorService wsCallBackExecutor; + private ExecutorService subscriptionUpdateExecutor; @PostConstruct public void initExecutor() { - wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); + subscriptionUpdateExecutor = Executors.newWorkStealingPool(20); } @PreDestroy public void shutdownExecutor() { - if (wsCallBackExecutor != null) { - wsCallBackExecutor.shutdownNow(); + if (subscriptionUpdateExecutor != null) { + subscriptionUpdateExecutor.shutdownNow(); } } @@ -108,7 +97,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer * Since number of subscriptions is usually much less then number of devices that are pushing data. */ subscriptionsBySessionId.values().forEach(map -> map.values() - .forEach(sub -> pushSubscriptionToManagerService(sub, false))); + .forEach(sub -> pushSubscriptionToManagerService(sub, true))); } } @@ -148,7 +137,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer update.getLatestValues().forEach((key, value) -> attrSub.getKeyStates().put(key, value)); break; } - subscription.getUpdateConsumer().accept(sessionId, update); + subscriptionUpdateExecutor.submit(() -> subscription.getUpdateConsumer().accept(sessionId, update)); } callback.onSuccess(); } @@ -158,7 +147,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer TbSubscription subscription = subscriptionsBySessionId .getOrDefault(sessionId, Collections.emptyMap()).get(update.getSubscriptionId()); if (subscription != null && subscription.getType() == TbSubscriptionType.ALARMS) { - subscription.getUpdateConsumer().accept(sessionId, update); + subscriptionUpdateExecutor.submit(() -> subscription.getUpdateConsumer().accept(sessionId, update)); } callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index bc4099e81b..468c3c26b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -105,10 +105,10 @@ public abstract class AbstractSubscriptionService implements ApplicationListener } } - protected void addWsCallback(ListenableFuture> saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback>() { + protected void addWsCallback(ListenableFuture saveFuture, Consumer callback) { + Futures.addCallback(saveFuture, new FutureCallback() { @Override - public void onSuccess(@Nullable List result) { + public void onSuccess(@Nullable T result) { callback.accept(null); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index aecd888dce..5b3ad05ee9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -20,12 +20,12 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -35,17 +35,19 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import javax.annotation.Nullable; @@ -59,12 +61,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.function.Consumer; -import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -76,6 +74,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final AttributesService attrService; private final TimeseriesService tsService; private final EntityViewService entityViewService; + private final TbApiUsageClient apiUsageClient; + private final TbApiUsageStateService apiUsageStateService; private ExecutorService tsCallBackExecutor; @@ -83,11 +83,15 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TimeseriesService tsService, EntityViewService entityViewService, TbClusterService clusterService, - PartitionService partitionService) { + PartitionService partitionService, + TbApiUsageClient apiUsageClient, + TbApiUsageStateService apiUsageStateService) { super(clusterService, partitionService); this.attrService = attrService; this.tsService = tsService; this.entityViewService = entityViewService; + this.apiUsageClient = apiUsageClient; + this.apiUsageStateService = apiUsageStateService; } @PostConstruct @@ -116,7 +120,36 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { - ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); + checkInternalEntity(entityId); + boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null; + if (sysTenant || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { + saveAndNotifyInternal(tenantId, entityId, ts, ttl, new FutureCallback() { + @Override + public void onSuccess(Integer result) { + if (!sysTenant && result != null && result > 0) { + apiUsageClient.report(tenantId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + } + callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }); + } else { + callback.onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); + } + } + + @Override + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { + saveAndNotifyInternal(tenantId, entityId, ts, 0L, callback); + } + + @Override + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { + ListenableFuture saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { @@ -141,9 +174,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer Optional tsKvEntry = entries.stream() .filter(entry -> entry.getTs() > startTs && entry.getTs() <= endTs) .max(Comparator.comparingLong(TsKvEntry::getTs)); - if (tsKvEntry.isPresent()) { - entityViewLatest.add(tsKvEntry.get()); - } + tsKvEntry.ifPresent(entityViewLatest::add); } } if (!entityViewLatest.isEmpty()) { @@ -176,29 +207,53 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { + checkInternalEntity(entityId); + saveAndNotifyInternal(tenantId, entityId, scope, attributes, notifyDevice, callback); + } + + @Override + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); - addMainCallback(saveFuture, callback); + addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); } @Override public void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { + checkInternalEntity(entityId); + saveLatestAndNotifyInternal(tenantId, entityId, ts, callback); + } + + @Override + public void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); - addMainCallback(saveFuture, callback); + addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } @Override public void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback) { + checkInternalEntity(entityId); + deleteAndNotifyInternal(tenantId, entityId, scope, keys, callback); + } + + @Override + public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback) { ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); - addMainCallback(deleteFuture, callback); + addVoidCallback(deleteFuture, callback); addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys)); } @Override public void deleteLatest(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback) { + checkInternalEntity(entityId); + deleteLatestInternal(tenantId, entityId, keys, callback); + } + + @Override + public void deleteLatestInternal(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback) { ListenableFuture> deleteFuture = tsService.removeLatest(tenantId, entityId, keys); - addMainCallback(deleteFuture, callback); + addVoidCallback(deleteFuture, callback); } @Override @@ -283,7 +338,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { + private void addVoidCallback(ListenableFuture saveFuture, final FutureCallback callback) { Futures.addCallback(saveFuture, new FutureCallback() { @Override public void onSuccess(@Nullable S result) { @@ -296,4 +351,25 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } }, tsCallBackExecutor); } + + private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { + Futures.addCallback(saveFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable S result) { + callback.onSuccess(result); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }, tsCallBackExecutor); + } + + private void checkInternalEntity(EntityId entityId) { + if (EntityType.API_USAGE_STATE.equals(entityId.getEntityType())) { + throw new RuntimeException("Can't update API Usage State!"); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java new file mode 100644 index 0000000000..1ca2cdeacb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry; + +import com.google.common.util.concurrent.FutureCallback; +import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +/** + * Created by ashvayka on 27.03.18. + */ +public interface InternalTelemetryService extends RuleEngineTelemetryService { + + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback); + + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback); + + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback); + + void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback); + + void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback); + + void deleteLatestInternal(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback); + + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index d652a3f1c5..a88a378cd7 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -22,6 +22,6 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; /** * Created by ashvayka on 27.03.18. */ -public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener { +public interface TelemetrySubscriptionService extends InternalTelemetryService, ApplicationListener { } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 7d41190ac0..98f5708bad 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -23,14 +23,17 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; -import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; +import org.thingsboard.server.common.data.device.credentials.ProvisionDeviceCredentialsData; +import org.thingsboard.server.common.data.device.profile.ProvisionDeviceProfileCredentials; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -44,18 +47,23 @@ import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.dao.device.DeviceCredentialsService; -import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceProvisionService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.device.provision.ProvisionRequest; +import org.thingsboard.server.dao.device.provision.ProvisionResponse; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.dao.tenant.TenantProfileService; -import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceCredentialsProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsgOrBuilder; +import org.thingsboard.server.gen.transport.TransportProtos.ProvisionResponseStatus; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; @@ -63,7 +71,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenR import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.dao.device.provision.ProvisionFailedException; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.state.DeviceStateService; @@ -83,10 +95,9 @@ public class DefaultTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); - //TODO: Constructor dependencies; - private final DeviceProfileService deviceProfileService; - private final TenantService tenantService; - private final TenantProfileService tenantProfileService; + private final TbDeviceProfileCache deviceProfileCache; + private final TbTenantProfileCache tenantProfileCache; + private final TbApiUsageStateService apiUsageStateService; private final DeviceService deviceService; private final RelationService relationService; private final DeviceCredentialsService deviceCredentialsService; @@ -94,18 +105,19 @@ public class DefaultTransportApiService implements TransportApiService { private final DbCallbackExecutorService dbCallbackExecutorService; private final TbClusterService tbClusterService; private final DataDecodingEncodingService dataDecodingEncodingService; - + private final DeviceProvisionService deviceProvisionService; private final ConcurrentMap deviceCreationLocks = new ConcurrentHashMap<>(); - public DefaultTransportApiService(DeviceProfileService deviceProfileService, TenantService tenantService, - TenantProfileService tenantProfileService, DeviceService deviceService, + public DefaultTransportApiService(TbDeviceProfileCache deviceProfileCache, + TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, DeviceService deviceService, RelationService relationService, DeviceCredentialsService deviceCredentialsService, DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService, - TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService) { - this.deviceProfileService = deviceProfileService; - this.tenantService = tenantService; - this.tenantProfileService = tenantProfileService; + TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService, + DeviceProvisionService deviceProvisionService) { + this.deviceProfileCache = deviceProfileCache; + this.tenantProfileCache = tenantProfileCache; + this.apiUsageStateService = apiUsageStateService; this.deviceService = deviceService; this.relationService = relationService; this.deviceCredentialsService = deviceCredentialsService; @@ -113,6 +125,7 @@ public class DefaultTransportApiService implements TransportApiService { this.dbCallbackExecutorService = dbCallbackExecutorService; this.tbClusterService = tbClusterService; this.dataDecodingEncodingService = dataDecodingEncodingService; + this.deviceProvisionService = deviceProvisionService; } @Override @@ -133,11 +146,11 @@ public class DefaultTransportApiService implements TransportApiService { } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) { return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); - } else if (transportApiRequestMsg.hasGetTenantRoutingInfoRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getGetTenantRoutingInfoRequestMsg()), + } else if (transportApiRequestMsg.hasEntityProfileRequestMsg()) { + return Futures.transform(handle(transportApiRequestMsg.getEntityProfileRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); - } else if (transportApiRequestMsg.hasGetDeviceProfileRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getGetDeviceProfileRequestMsg()), + } else if (transportApiRequestMsg.hasProvisionDeviceRequestMsg()) { + return Futures.transform(handle(transportApiRequestMsg.getProvisionDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } return Futures.transform(getEmptyTransportApiResponseFuture(), @@ -155,22 +168,27 @@ public class DefaultTransportApiService implements TransportApiService { } private ListenableFuture validateCredentials(TransportProtos.ValidateBasicMqttCredRequestMsg mqtt) { - DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByCredentialsId(mqtt.getUserName()); - if (credentials != null) { - if (credentials.getCredentialsType() == DeviceCredentialsType.ACCESS_TOKEN) { - return getDeviceInfo(credentials.getDeviceId(), credentials); - } else if (credentials.getCredentialsType() == DeviceCredentialsType.MQTT_BASIC) { - if (!checkMqttCredentials(mqtt, credentials)) { - credentials = null; + DeviceCredentials credentials = null; + if (!StringUtils.isEmpty(mqtt.getUserName())) { + credentials = deviceCredentialsService.findDeviceCredentialsByCredentialsId(mqtt.getUserName()); + if (credentials != null) { + if (credentials.getCredentialsType() == DeviceCredentialsType.ACCESS_TOKEN) { + return getDeviceInfo(credentials.getDeviceId(), credentials); + } else if (credentials.getCredentialsType() == DeviceCredentialsType.MQTT_BASIC) { + if (!checkMqttCredentials(mqtt, credentials)) { + credentials = null; + } + } else { + return getEmptyTransportApiResponseFuture(); } } - } - if (credentials == null) { - credentials = checkMqttCredentials(mqtt, EncryptionUtil.getSha3Hash("|", mqtt.getClientId(), mqtt.getUserName())); if (credentials == null) { - credentials = checkMqttCredentials(mqtt, EncryptionUtil.getSha3Hash(mqtt.getClientId())); + credentials = checkMqttCredentials(mqtt, EncryptionUtil.getSha3Hash("|", mqtt.getClientId(), mqtt.getUserName())); } } + if (credentials == null) { + credentials = checkMqttCredentials(mqtt, EncryptionUtil.getSha3Hash(mqtt.getClientId())); + } if (credentials != null) { return getDeviceInfo(credentials.getDeviceId(), credentials); } else { @@ -225,7 +243,7 @@ public class DefaultTransportApiService implements TransportApiService { device.setName(requestMsg.getDeviceName()); device.setType(requestMsg.getDeviceType()); device.setCustomerId(gateway.getCustomerId()); - DeviceProfile deviceProfile = deviceProfileService.findOrCreateDeviceProfile(gateway.getTenantId(), requestMsg.getDeviceType()); + DeviceProfile deviceProfile = deviceProfileCache.findOrCreateDeviceProfile(gateway.getTenantId(), requestMsg.getDeviceType()); device.setDeviceProfileId(deviceProfile.getId()); device = deviceService.saveDevice(device); relationService.saveRelationAsync(TenantId.SYS_TENANT_ID, new EntityRelation(gateway.getId(), device.getId(), "Created")); @@ -245,7 +263,7 @@ public class DefaultTransportApiService implements TransportApiService { } GetOrCreateDeviceFromGatewayResponseMsg.Builder builder = GetOrCreateDeviceFromGatewayResponseMsg.newBuilder() .setDeviceInfo(getDeviceInfoProto(device)); - DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileById(device.getTenantId(), device.getDeviceProfileId()); + DeviceProfile deviceProfile = deviceProfileCache.get(device.getTenantId(), device.getDeviceProfileId()); if (deviceProfile != null) { builder.setProfileBody(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile))); } else { @@ -263,25 +281,70 @@ public class DefaultTransportApiService implements TransportApiService { }, dbCallbackExecutorService); } - private ListenableFuture handle(GetTenantRoutingInfoRequestMsg requestMsg) { - TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB())); - // TODO: Tenant Profile from cache - ListenableFuture tenantProfileFuture = - Futures.transform(tenantService.findTenantByIdAsync(TenantId.SYS_TENANT_ID, tenantId), tenant -> - tenantProfileService.findTenantProfileById(TenantId.SYS_TENANT_ID, tenant.getTenantProfileId()), dbCallbackExecutorService); - return Futures.transform(tenantProfileFuture, tenantProfile -> TransportApiResponseMsg.newBuilder() - .setGetTenantRoutingInfoResponseMsg(GetTenantRoutingInfoResponseMsg.newBuilder().setIsolatedTbCore(tenantProfile.isIsolatedTbCore()) - .setIsolatedTbRuleEngine(tenantProfile.isIsolatedTbRuleEngine()).build()).build(), dbCallbackExecutorService); + private ListenableFuture handle(ProvisionDeviceRequestMsg requestMsg) { + ListenableFuture provisionResponseFuture = null; + try { + provisionResponseFuture = Futures.immediateFuture(deviceProvisionService.provisionDevice( + new ProvisionRequest( + requestMsg.getDeviceName(), + requestMsg.getCredentialsType() != null ? DeviceCredentialsType.valueOf(requestMsg.getCredentialsType().name()) : null, + new ProvisionDeviceCredentialsData(requestMsg.getCredentialsDataProto().getValidateDeviceTokenRequestMsg().getToken(), + requestMsg.getCredentialsDataProto().getValidateBasicMqttCredRequestMsg().getClientId(), + requestMsg.getCredentialsDataProto().getValidateBasicMqttCredRequestMsg().getUserName(), + requestMsg.getCredentialsDataProto().getValidateBasicMqttCredRequestMsg().getPassword(), + requestMsg.getCredentialsDataProto().getValidateDeviceX509CertRequestMsg().getHash()), + new ProvisionDeviceProfileCredentials( + requestMsg.getProvisionDeviceCredentialsMsg().getProvisionDeviceKey(), + requestMsg.getProvisionDeviceCredentialsMsg().getProvisionDeviceSecret())))); + } catch (ProvisionFailedException e) { + return Futures.immediateFuture(getTransportApiResponseMsg( + new DeviceCredentials(), + TransportProtos.ProvisionResponseStatus.valueOf(e.getMessage()))); + } + return Futures.transform(provisionResponseFuture, provisionResponse -> getTransportApiResponseMsg(provisionResponse.getDeviceCredentials(), TransportProtos.ProvisionResponseStatus.SUCCESS), + dbCallbackExecutorService); + } + + private TransportApiResponseMsg getTransportApiResponseMsg(DeviceCredentials deviceCredentials, TransportProtos.ProvisionResponseStatus status) { + if (!status.equals(ProvisionResponseStatus.SUCCESS)) { + return TransportApiResponseMsg.newBuilder().setProvisionDeviceResponseMsg(TransportProtos.ProvisionDeviceResponseMsg.newBuilder().setStatus(status).build()).build(); + } + TransportProtos.ProvisionDeviceResponseMsg.Builder provisionResponse = TransportProtos.ProvisionDeviceResponseMsg.newBuilder() + .setCredentialsType(TransportProtos.CredentialsType.valueOf(deviceCredentials.getCredentialsType().name())) + .setStatus(status); + switch (deviceCredentials.getCredentialsType()){ + case ACCESS_TOKEN: + provisionResponse.setCredentialsValue(deviceCredentials.getCredentialsId()); + break; + case MQTT_BASIC: + case X509_CERTIFICATE: + provisionResponse.setCredentialsValue(deviceCredentials.getCredentialsValue()); + break; + } + + return TransportApiResponseMsg.newBuilder() + .setProvisionDeviceResponseMsg(provisionResponse.build()) + .build(); } - private ListenableFuture handle(TransportProtos.GetDeviceProfileRequestMsg requestMsg) { - DeviceProfileId profileId = new DeviceProfileId(new UUID(requestMsg.getProfileIdMSB(), requestMsg.getProfileIdLSB())); - DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileById(TenantId.SYS_TENANT_ID, profileId); - return Futures.immediateFuture(TransportApiResponseMsg.newBuilder() - .setGetDeviceProfileResponseMsg( - TransportProtos.GetDeviceProfileResponseMsg.newBuilder() - .setData(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile))) - .build()).build()); + private ListenableFuture handle(GetEntityProfileRequestMsg requestMsg) { + EntityType entityType = EntityType.valueOf(requestMsg.getEntityType()); + UUID entityUuid = new UUID(requestMsg.getEntityIdMSB(), requestMsg.getEntityIdLSB()); + GetEntityProfileResponseMsg.Builder builder = GetEntityProfileResponseMsg.newBuilder(); + if (entityType.equals(EntityType.DEVICE_PROFILE)) { + DeviceProfileId deviceProfileId = new DeviceProfileId(entityUuid); + DeviceProfile deviceProfile = deviceProfileCache.find(deviceProfileId); + builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile))); + } else if (entityType.equals(EntityType.TENANT)) { + TenantId tenantId = new TenantId(entityUuid); + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + ApiUsageState state = apiUsageStateService.getApiUsageState(tenantId); + builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile))); + builder.setApiState(ByteString.copyFrom(dataDecodingEncodingService.encode(state))); + } else { + throw new RuntimeException("Invalid entity profile request: " + entityType); + } + return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(builder).build()); } private ListenableFuture getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { @@ -293,7 +356,7 @@ public class DefaultTransportApiService implements TransportApiService { try { ValidateDeviceCredentialsResponseMsg.Builder builder = ValidateDeviceCredentialsResponseMsg.newBuilder(); builder.setDeviceInfo(getDeviceInfoProto(device)); - DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileById(device.getTenantId(), device.getDeviceProfileId()); + DeviceProfile deviceProfile = deviceProfileCache.get(device.getTenantId(), device.getDeviceProfileId()); if (deviceProfile != null) { builder.setProfileBody(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile))); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index a191a3167e..44fbf17a42 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.thingsboard.server.common.stats.MessagesStats; import org.thingsboard.server.common.stats.StatsFactory; @@ -90,6 +91,7 @@ public class TbCoreTransportApiService { } @EventListener(ApplicationReadyEvent.class) + @Order(value = 2) public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { log.info("Received application ready event. Starting polling for events."); transportApiTemplate.init(transportApiService); diff --git a/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java b/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java index ed13ca603d..e2d9302d79 100644 --- a/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java @@ -49,12 +49,43 @@ public class MiscUtils { } public static String constructBaseUrl(HttpServletRequest request) { - String scheme = request.getScheme(); + return String.format("%s://%s:%d", + getScheme(request), + getDomainName(request), + getPort(request)); + } + public static String getScheme(HttpServletRequest request){ + String scheme = request.getScheme(); String forwardedProto = request.getHeader("x-forwarded-proto"); if (forwardedProto != null) { scheme = forwardedProto; } + return scheme; + } + + public static String getDomainName(HttpServletRequest request){ + return request.getServerName(); + } + + public static String getDomainNameAndPort(HttpServletRequest request){ + String domainName = getDomainName(request); + String scheme = getScheme(request); + int port = MiscUtils.getPort(request); + if (needsPort(scheme, port)) { + domainName += ":" + port; + } + return domainName; + } + + private static boolean needsPort(String scheme, int port) { + boolean isHttpDefault = "http".equals(scheme.toLowerCase()) && port == 80; + boolean isHttpsDefault = "https".equals(scheme.toLowerCase()) && port == 443; + return !isHttpDefault && !isHttpsDefault; + } + + public static int getPort(HttpServletRequest request){ + String forwardedProto = request.getHeader("x-forwarded-proto"); int serverPort = request.getServerPort(); if (request.getHeader("x-forwarded-port") != null) { @@ -72,11 +103,6 @@ public class MiscUtils { break; } } - - String baseUrl = String.format("%s://%s:%d", - scheme, - request.getServerName(), - serverPort); - return baseUrl; + return serverPort; } } diff --git a/application/src/main/resources/i18n/messages.properties b/application/src/main/resources/i18n/messages.properties index a34582e132..bdbd64384b 100644 --- a/application/src/main/resources/i18n/messages.properties +++ b/application/src/main/resources/i18n/messages.properties @@ -3,4 +3,5 @@ activation.subject=Your account activation on Thingsboard account.activated.subject=Thingsboard - your account has been activated reset.password.subject=Thingsboard - Password reset has been requested password.was.reset.subject=Thingsboard - your account password has been reset -account.lockout.subject=Thingsboard - User account has been lockout \ No newline at end of file +account.lockout.subject=Thingsboard - User account has been lockout +api.usage.state=Thingsboard - Account limits \ No newline at end of file diff --git a/application/src/main/resources/templates/state.disabled.ftl b/application/src/main/resources/templates/state.disabled.ftl new file mode 100644 index 0000000000..2e341dbe20 --- /dev/null +++ b/application/src/main/resources/templates/state.disabled.ftl @@ -0,0 +1,145 @@ +<#-- + + Copyright © 2016-2020 The Thingsboard Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + + + + Thingsboard - Api Usage State + + + + + + + + + + + + + + + + + + Your ThingsBoard account feature was disabled + + + + + We have disabled the ${apiFeature} for your account because ThingsBoard has already ${apiLimitValueLabel}. + + + + + Please contact your system administrator to resolve the issue. + + + — The ThingsBoard + + + + + + + + + + + This email was sent to ${targetEmail} by ThingsBoard. + + + + + diff --git a/application/src/main/resources/templates/state.enabled.ftl b/application/src/main/resources/templates/state.enabled.ftl new file mode 100644 index 0000000000..e03f38bba3 --- /dev/null +++ b/application/src/main/resources/templates/state.enabled.ftl @@ -0,0 +1,142 @@ +<#-- + + Copyright © 2016-2020 The Thingsboard Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + + + + Thingsboard - Api Usage State + + + + + + + + + + + + + + + + + + Your ThingsBoard account feature was enabled + + + + + We have enabled the ${apiFeature} for your account and ThingsBoard is already able to ${apiLabel} messages. + + + + + — The ThingsBoard + + + + + + + + + + + This email was sent to ${targetEmail} by ThingsBoard. + + + + + diff --git a/application/src/main/resources/templates/state.warning.ftl b/application/src/main/resources/templates/state.warning.ftl new file mode 100644 index 0000000000..67fc557e87 --- /dev/null +++ b/application/src/main/resources/templates/state.warning.ftl @@ -0,0 +1,145 @@ +<#-- + + Copyright © 2016-2020 The Thingsboard Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + + + + Thingsboard - Api Usage State + + + + + + + + + + + + + + + + + + Warning: your ThingsBoard account feature may be disabled soon + + + + + ThingsBoard has already ${apiValueLabel}.${apiFeature} will be disabled for your account once the limit will be reached. + + + + + Please contact your system administrator to resolve the issue. + + + — The ThingsBoard + + + + + + + + + + + This email was sent to ${targetEmail} by ThingsBoard. + + + + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index fbbff1573a..d4da833e40 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -113,62 +113,10 @@ security: basic: enabled: "${SECURITY_BASIC_ENABLED:false}" oauth2: - # Enable/disable OAuth 2 login functionality - # For details please refer to https://thingsboard.io/docs/user-guide/oauth-2-support/ - enabled: "${SECURITY_OAUTH2_ENABLED:false}" # Redirect URL where access code from external user management system will be processed loginProcessingUrl: "${SECURITY_OAUTH2_LOGIN_PROCESSING_URL:/login/oauth2/code/}" - # List of SSO clients - clients: - default: - # Label that going to be show on login button - 'Login with {loginButtonLabel}' - loginButtonLabel: "${SECURITY_OAUTH2_DEFAULT_LOGIN_BUTTON_LABEL:Default}" - # Icon that going to be show on login button. Material design icon ID (https://material.angularjs.org/latest/api/directive/mdIcon) - loginButtonIcon: "${SECURITY_OAUTH2_DEFAULT_LOGIN_BUTTON_ICON:}" - clientName: "${SECURITY_OAUTH2_DEFAULT_CLIENT_NAME:ClientName}" - clientId: "${SECURITY_OAUTH2_DEFAULT_CLIENT_ID:}" - clientSecret: "${SECURITY_OAUTH2_DEFAULT_CLIENT_SECRET:}" - accessTokenUri: "${SECURITY_OAUTH2_DEFAULT_ACCESS_TOKEN_URI:}" - authorizationUri: "${SECURITY_OAUTH2_DEFAULT_AUTHORIZATION_URI:}" - scope: "${SECURITY_OAUTH2_DEFAULT_SCOPE:}" - # Redirect URL that must be in sync with 'security.oauth2.loginProcessingUrl', but domain name added - redirectUriTemplate: "${SECURITY_OAUTH2_DEFAULT_REDIRECT_URI_TEMPLATE:http://localhost:8080/login/oauth2/code/}" - jwkSetUri: "${SECURITY_OAUTH2_DEFAULT_JWK_SET_URI:}" - # 'authorization_code', 'implicit', 'refresh_token' or 'client_credentials' - authorizationGrantType: "${SECURITY_OAUTH2_DEFAULT_AUTHORIZATION_GRANT_TYPE:authorization_code}" - clientAuthenticationMethod: "${SECURITY_OAUTH2_DEFAULT_CLIENT_AUTHENTICATION_METHOD:post}" # basic or post - userInfoUri: "${SECURITY_OAUTH2_DEFAULT_USER_INFO_URI:}" - userNameAttributeName: "${SECURITY_OAUTH2_DEFAULT_USER_NAME_ATTRIBUTE_NAME:email}" - mapperConfig: - # Allows to create user if it not exists - allowUserCreation: "${SECURITY_OAUTH2_DEFAULT_MAPPER_ALLOW_USER_CREATION:true}" - # Allows user to setup ThingsBoard internal password and login over default Login window - activateUser: "${SECURITY_OAUTH2_DEFAULT_MAPPER_ACTIVATE_USER:false}" - # Mapper type of converter from external user into internal - 'basic' or 'custom' - type: "${SECURITY_OAUTH2_DEFAULT_MAPPER_TYPE:basic}" - basic: - # Key from attributes of external user object to use as email - emailAttributeKey: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_EMAIL_ATTRIBUTE_KEY:email}" - firstNameAttributeKey: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_FIRST_NAME_ATTRIBUTE_KEY:}" - lastNameAttributeKey: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_LAST_NAME_ATTRIBUTE_KEY:}" - # Strategy for generating Tenant from external user object - 'domain', 'email' or 'custom' - # 'domain' - name of the Tenant will be extracted as domain from the email of the user - # 'email' - name of the Tenant will email of the user - # 'custom' - please configure 'tenantNamePattern' for custom mapping - tenantNameStrategy: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_TENANT_NAME_STRATEGY:domain}" - # %{attribute_key} as placeholder for attribute value of attributes of external user object - tenantNamePattern: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_TENANT_NAME_PATTERN:}" - # If this field is not empty, user will be created as a user under defined Customer - # %{attribute_key} as placeholder for attribute value of attributes of external user object - customerNamePattern: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_CUSTOMER_NAME_PATTERN:}" - # If this field is not empty, user will be created with default defined Dashboard - defaultDashboardName: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_DEFAULT_DASHBOARD_NAME:}" - # If this field is set 'true' along with non-empty 'defaultDashboardName', user will start from the defined Dashboard in fullscreen mode - alwaysFullScreen: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_ALWAYS_FULL_SCREEN:false}" - custom: - url: "${SECURITY_OAUTH2_DEFAULT_MAPPER_CUSTOM_URL:}" - username: "${SECURITY_OAUTH2_DEFAULT_MAPPER_CUSTOM_USERNAME:}" - password: "${SECURITY_OAUTH2_DEFAULT_MAPPER_CUSTOM_PASSWORD:}" + githubMapper: + emailUrl: "${SECURITY_OAUTH2_GITHUB_MAPPER_EMAIL_URL_KEY:https://api.github.com/user/emails}" # Dashboard parameters dashboard: @@ -337,8 +285,12 @@ actors: js_thread_pool_size: "${ACTORS_RULE_JS_THREAD_POOL_SIZE:50}" # Specify thread pool size for mail sender executor service mail_thread_pool_size: "${ACTORS_RULE_MAIL_THREAD_POOL_SIZE:50}" + # Specify thread pool size for sms sender executor service + sms_thread_pool_size: "${ACTORS_RULE_SMS_THREAD_POOL_SIZE:50}" # Whether to allow usage of system mail service for rules allow_system_mail_service: "${ACTORS_RULE_ALLOW_SYSTEM_MAIL_SERVICE:true}" + # Whether to allow usage of system sms service for rules + allow_system_sms_service: "${ACTORS_RULE_ALLOW_SYSTEM_SMS_SERVICE:true}" # Specify thread pool size for external call service external_call_thread_pool_size: "${ACTORS_RULE_EXTERNAL_CALL_THREAD_POOL_SIZE:50}" chain: @@ -562,7 +514,7 @@ js: remote: # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${REMOTE_JS_SANDBOX_MAX_ERRORS:3}" - # Maximum time in seconds for black listed function to stay in the list. + # Maximum time in seconds for black listed function to stay in 1:the list. max_black_list_duration_sec: "${REMOTE_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_REMOTE_STATS_ENABLED:false}" @@ -572,10 +524,6 @@ transport: sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" - rate_limits: - enabled: "${TB_TRANSPORT_RATE_LIMITS_ENABLED:false}" - tenant: "${TB_TRANSPORT_RATE_LIMITS_TENANT:1000:1,20000:60}" - device: "${TB_TRANSPORT_RATE_LIMITS_DEVICE:10:1,300:60}" json: # Cast String data types to Numeric if possible when processing Telemetry/Attributes JSON type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" @@ -674,6 +622,7 @@ queue: linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:0}" max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" @@ -756,6 +705,7 @@ queue: poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" @@ -848,7 +798,6 @@ metrics: # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" - management: endpoints: web: diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 65ec10db91..6c45fe327a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -34,12 +33,7 @@ import org.junit.Rule; import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; -import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootContextLoader; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.converter.HttpMessageConverter; @@ -47,10 +41,6 @@ import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.mock.http.MockHttpInputMessage; import org.springframework.mock.http.MockHttpOutputMessage; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.ResultActions; @@ -59,7 +49,6 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilde import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.context.WebApplicationContext; -import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; @@ -70,10 +59,13 @@ import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileCon import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.MqttTopics; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.security.Authority; @@ -331,7 +323,7 @@ public abstract class AbstractWebTest { } } - protected DeviceProfile createDeviceProfile(String name) { + protected DeviceProfile createDeviceProfile(String name, DeviceProfileTransportConfiguration deviceProfileTransportConfiguration) { DeviceProfile deviceProfile = new DeviceProfile(); deviceProfile.setName(name); deviceProfile.setType(DeviceProfileType.DEFAULT); @@ -339,15 +331,34 @@ public abstract class AbstractWebTest { deviceProfile.setDescription(name + " Test"); DeviceProfileData deviceProfileData = new DeviceProfileData(); DefaultDeviceProfileConfiguration configuration = new DefaultDeviceProfileConfiguration(); - DefaultDeviceProfileTransportConfiguration transportConfiguration = new DefaultDeviceProfileTransportConfiguration(); deviceProfileData.setConfiguration(configuration); - deviceProfileData.setTransportConfiguration(transportConfiguration); + if (deviceProfileTransportConfiguration != null) { + deviceProfileData.setTransportConfiguration(deviceProfileTransportConfiguration); + } else { + deviceProfileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration()); + } deviceProfile.setProfileData(deviceProfileData); deviceProfile.setDefault(false); deviceProfile.setDefaultRuleChainId(null); return deviceProfile; } + protected MqttDeviceProfileTransportConfiguration createMqttDeviceProfileTransportConfiguration(TransportPayloadTypeConfiguration transportPayloadTypeConfiguration) { + MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = new MqttDeviceProfileTransportConfiguration(); + mqttDeviceProfileTransportConfiguration.setDeviceTelemetryTopic(MqttTopics.DEVICE_TELEMETRY_TOPIC); + mqttDeviceProfileTransportConfiguration.setDeviceTelemetryTopic(MqttTopics.DEVICE_ATTRIBUTES_TOPIC); + mqttDeviceProfileTransportConfiguration.setTransportPayloadTypeConfiguration(transportPayloadTypeConfiguration); + return mqttDeviceProfileTransportConfiguration; + } + + protected ProtoTransportPayloadConfiguration createProtoTransportPayloadConfiguration(String deviceAttributesProtoSchema, String deviceTelemetryProtoSchema) { + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = new ProtoTransportPayloadConfiguration(); + protoTransportPayloadConfiguration.setDeviceAttributesProtoSchema(deviceAttributesProtoSchema); + protoTransportPayloadConfiguration.setDeviceTelemetryProtoSchema(deviceTelemetryProtoSchema); + return protoTransportPayloadConfiguration; + } + + protected ResultActions doGet(String urlTemplate, Object... urlVariables) throws Exception { MockHttpServletRequestBuilder getRequest = get(urlTemplate, urlVariables); setJwtToken(getRequest); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java index b2334d7c46..841e899473 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java @@ -16,6 +16,12 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -28,6 +34,10 @@ import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; @@ -35,9 +45,13 @@ import org.thingsboard.server.common.data.security.Authority; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; public abstract class BaseDeviceProfileControllerTest extends AbstractControllerTest { @@ -77,7 +91,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController @Test public void testSaveDeviceProfile() throws Exception { - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", null); DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); Assert.assertNotNull(savedDeviceProfile); Assert.assertNotNull(savedDeviceProfile.getId()); @@ -95,7 +109,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController @Test public void testFindDeviceProfileById() throws Exception { - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", null); DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); DeviceProfile foundDeviceProfile = doGet("/api/deviceProfile/"+savedDeviceProfile.getId().getId().toString(), DeviceProfile.class); Assert.assertNotNull(foundDeviceProfile); @@ -104,7 +118,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController @Test public void testFindDeviceProfileInfoById() throws Exception { - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", null); DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); DeviceProfileInfo foundDeviceProfileInfo = doGet("/api/deviceProfileInfo/"+savedDeviceProfile.getId().getId().toString(), DeviceProfileInfo.class); Assert.assertNotNull(foundDeviceProfileInfo); @@ -126,7 +140,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController @Test public void testSetDefaultDeviceProfile() throws Exception { - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile 1"); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile 1", null); DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); DeviceProfile defaultDeviceProfile = doPost("/api/deviceProfile/"+savedDeviceProfile.getId().getId().toString()+"/default", null, DeviceProfile.class); Assert.assertNotNull(defaultDeviceProfile); @@ -146,17 +160,28 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController @Test public void testSaveDeviceProfileWithSameName() throws Exception { - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", null); doPost("/api/deviceProfile", deviceProfile).andExpect(status().isOk()); - DeviceProfile deviceProfile2 = this.createDeviceProfile("Device Profile"); + DeviceProfile deviceProfile2 = this.createDeviceProfile("Device Profile", null); doPost("/api/deviceProfile", deviceProfile2).andExpect(status().isBadRequest()) .andExpect(statusReason(containsString("Device profile with such name already exists"))); } + @Test + public void testSaveDeviceProfileWithSameProvisionDeviceKey() throws Exception { + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", null); + deviceProfile.setProvisionDeviceKey("testProvisionDeviceKey"); + doPost("/api/deviceProfile", deviceProfile).andExpect(status().isOk()); + DeviceProfile deviceProfile2 = this.createDeviceProfile("Device Profile 2", null); + deviceProfile2.setProvisionDeviceKey("testProvisionDeviceKey"); + doPost("/api/deviceProfile", deviceProfile2).andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Device profile with such provision device key already exists"))); + } + @Ignore @Test public void testChangeDeviceProfileTypeWithExistingDevices() throws Exception { - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", null); DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); Device device = new Device(); device.setName("Test device"); @@ -171,7 +196,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController @Test public void testChangeDeviceProfileTransportTypeWithExistingDevices() throws Exception { - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", null); DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); Device device = new Device(); device.setName("Test device"); @@ -185,7 +210,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController @Test public void testDeleteDeviceProfileWithExistingDevice() throws Exception { - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", null); DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); Device device = new Device(); @@ -202,7 +227,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController @Test public void testDeleteDeviceProfile() throws Exception { - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", null); DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); doDelete("/api/deviceProfile/" + savedDeviceProfile.getId().getId().toString()) @@ -223,7 +248,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController deviceProfiles.addAll(pageData.getData()); for (int i=0;i<28;i++) { - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"+i); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"+i, null); deviceProfiles.add(doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class)); } @@ -268,7 +293,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController deviceProfiles.addAll(deviceProfilePageData.getData()); for (int i=0;i<28;i++) { - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"+i); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"+i, null); deviceProfiles.add(doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class)); } @@ -306,4 +331,341 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController Assert.assertEquals(1, pageData.getTotalElements()); } + @Test + public void testSaveProtoDeviceProfileWithInvalidProtoFile() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "message SchemaValidationTest {\n" + + " required int32 parameter = 1;\n" + + "}", "[Transport Configuration] failed to parse attributes proto schema due to: Syntax error in :6:4: 'required' label forbidden in proto3 field declarations"); + } + + @Test + public void testSaveProtoDeviceProfileWithInvalidProtoSyntax() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto2\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "message SchemaValidationTest {\n" + + " required int32 parameter = 1;\n" + + "}", "[Transport Configuration] invalid schema syntax: proto2 for attributes proto schema provided! Only proto3 allowed!"); + } + + @Test + public void testSaveProtoDeviceProfileOptionsNotSupported() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "option java_package = \"com.test.schemavalidation\";\n" + + "option java_multiple_files = true;\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "message SchemaValidationTest {\n" + + " int32 parameter = 1;\n" + + "}", "[Transport Configuration] invalid attributes proto schema provided! Schema options don't support!"); + } + + @Test + public void testSaveProtoDeviceProfilePublicImportsNotSupported() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "import public \"oldschema.proto\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "message SchemaValidationTest {\n" + + " int32 parameter = 1;\n" + + "}", "[Transport Configuration] invalid attributes proto schema provided! Schema public imports don't support!"); + } + + @Test + public void testSaveProtoDeviceProfileImportsNotSupported() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "import \"oldschema.proto\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "message SchemaValidationTest {\n" + + " int32 parameter = 1;\n" + + "}", "[Transport Configuration] invalid attributes proto schema provided! Schema imports don't support!"); + } + + @Test + public void testSaveProtoDeviceProfileExtendDeclarationsNotSupported() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "extend google.protobuf.MethodOptions {\n" + + " MyMessage my_method_option = 50007;\n" + + "}", "[Transport Configuration] invalid attributes proto schema provided! Schema extend declarations don't support!"); + } + + @Test + public void testSaveProtoDeviceProfileEnumOptionsNotSupported() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "enum testEnum {\n" + + " option allow_alias = true;\n" + + " DEFAULT = 0;\n" + + " STARTED = 1;\n" + + " RUNNING = 2;\n" + + "}\n" + + "\n" + + "message testMessage {\n" + + " int32 parameter = 1;\n" + + "}", "[Transport Configuration] invalid attributes proto schema provided! Enum definitions options are not supported!"); + } + + @Test + public void testSaveProtoDeviceProfileNoOneMessageTypeExists() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "enum testEnum {\n" + + " DEFAULT = 0;\n" + + " STARTED = 1;\n" + + " RUNNING = 2;\n" + + "}", "[Transport Configuration] invalid attributes proto schema provided! At least one Message definition should exists!"); + } + + @Test + public void testSaveProtoDeviceProfileMessageTypeOptionsNotSupported() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "message testMessage {\n" + + " option allow_alias = true;\n" + + " int32 parameter = 1;\n" + + "}", "[Transport Configuration] invalid attributes proto schema provided! Message definition options don't support!"); + } + + @Test + public void testSaveProtoDeviceProfileMessageTypeExtensionsNotSupported() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "message TestMessage {\n" + + " extensions 100 to 199;\n" + + "}", "[Transport Configuration] invalid attributes proto schema provided! Message definition extensions don't support!"); + } + + @Test + public void testSaveProtoDeviceProfileMessageTypeReservedElementsNotSupported() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "message Foo {\n" + + " reserved 2, 15, 9 to 11;\n" + + " reserved \"foo\", \"bar\";\n" + + "}", "[Transport Configuration] invalid attributes proto schema provided! Message definition reserved elements don't support!"); + } + + @Test + public void testSaveProtoDeviceProfileMessageTypeGroupsElementsNotSupported() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "message TestMessage {\n" + + " repeated group Result = 1 {\n" + + " string url = 2;\n" + + " string title = 3;\n" + + " repeated string snippets = 4;\n" + + " }\n" + + "}", "[Transport Configuration] invalid attributes proto schema provided! Message definition groups don't support!"); + } + + @Test + public void testSaveProtoDeviceProfileOneOfsGroupsElementsNotSupported() throws Exception { + testSaveDeviceProfileWithInvalidProtoSchema("syntax = \"proto3\";\n" + + "\n" + + "package schemavalidation;\n" + + "\n" + + "message SampleMessage {\n" + + " oneof test_oneof {\n" + + " string name = 1;\n" + + " group Result = 2 {\n" + + " \tstring url = 3;\n" + + " \tstring title = 4;\n" + + " \trepeated string snippets = 5;\n" + + " }\n" + + " }" + + "}", "[Transport Configuration] invalid attributes proto schema provided! OneOf definition groups don't support!"); + } + + @Test + public void testSaveProtoDeviceProfileWithMessageNestedTypes() throws Exception { + String schema = "syntax = \"proto3\";\n" + + "\n" + + "package testnested;\n" + + "\n" + + "message Outer {\n" + + " message MiddleAA {\n" + + " message Inner {\n" + + " int64 ival = 1;\n" + + " bool booly = 2;\n" + + " }\n" + + " Inner inner = 1;\n" + + " }\n" + + " message MiddleBB {\n" + + " message Inner {\n" + + " int32 ival = 1;\n" + + " bool booly = 2;\n" + + " }\n" + + " Inner inner = 1;\n" + + " }\n" + + " MiddleAA middleAA = 1;\n" + + " MiddleBB middleBB = 2;\n" + + "}"; + DynamicSchema dynamicSchema = getDynamicSchema(schema); + assertNotNull(dynamicSchema); + Set messageTypes = dynamicSchema.getMessageTypes(); + assertEquals(5, messageTypes.size()); + assertTrue(messageTypes.contains("testnested.Outer")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleAA")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleAA.Inner")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleBB")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleBB.Inner")); + + DynamicMessage.Builder middleAAInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); + Descriptors.Descriptor middleAAInnerMsgDescriptor = middleAAInnerMsgBuilder.getDescriptorForType(); + DynamicMessage middleAAInnerMsg = middleAAInnerMsgBuilder + .setField(middleAAInnerMsgDescriptor.findFieldByName("ival"), 1L) + .setField(middleAAInnerMsgDescriptor.findFieldByName("booly"), true) + .build(); + + DynamicMessage.Builder middleAAMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA"); + Descriptors.Descriptor middleAAMsgDescriptor = middleAAMsgBuilder.getDescriptorForType(); + DynamicMessage middleAAMsg = middleAAMsgBuilder + .setField(middleAAMsgDescriptor.findFieldByName("inner"), middleAAInnerMsg) + .build(); + + DynamicMessage.Builder middleBBInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); + Descriptors.Descriptor middleBBInnerMsgDescriptor = middleBBInnerMsgBuilder.getDescriptorForType(); + DynamicMessage middleBBInnerMsg = middleBBInnerMsgBuilder + .setField(middleBBInnerMsgDescriptor.findFieldByName("ival"), 0L) + .setField(middleBBInnerMsgDescriptor.findFieldByName("booly"), false) + .build(); + + DynamicMessage.Builder middleBBMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleBB"); + Descriptors.Descriptor middleBBMsgDescriptor = middleBBMsgBuilder.getDescriptorForType(); + DynamicMessage middleBBMsg = middleBBMsgBuilder + .setField(middleBBMsgDescriptor.findFieldByName("inner"), middleBBInnerMsg) + .build(); + + + DynamicMessage.Builder outerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer"); + Descriptors.Descriptor outerMsgBuilderDescriptor = outerMsgBuilder.getDescriptorForType(); + DynamicMessage outerMsg = outerMsgBuilder + .setField(outerMsgBuilderDescriptor.findFieldByName("middleAA"), middleAAMsg) + .setField(outerMsgBuilderDescriptor.findFieldByName("middleBB"), middleBBMsg) + .build(); + + assertEquals("{\n" + + " \"middleAA\": {\n" + + " \"inner\": {\n" + + " \"ival\": \"1\",\n" + + " \"booly\": true\n" + + " }\n" + + " },\n" + + " \"middleBB\": {\n" + + " \"inner\": {\n" + + " \"ival\": 0,\n" + + " \"booly\": false\n" + + " }\n" + + " }\n" + + "}", dynamicMsgToJson(outerMsgBuilderDescriptor, outerMsg.toByteArray())); + } + + @Test + public void testSaveProtoDeviceProfileWithMessageOneOfs() throws Exception { + String schema = "syntax = \"proto3\";\n" + + "\n" + + "package testoneofs;\n" + + "\n" + + "message SubMessage {\n" + + " repeated string name = 1;\n" + + "}\n" + + "\n" + + "message SampleMessage {\n" + + " oneof testOneOf {\n" + + " string name = 4;\n" + + " SubMessage subMessage = 9;\n" + + " }\n" + + "}"; + DynamicSchema dynamicSchema = getDynamicSchema(schema); + assertNotNull(dynamicSchema); + Set messageTypes = dynamicSchema.getMessageTypes(); + assertEquals(2, messageTypes.size()); + assertTrue(messageTypes.contains("testoneofs.SubMessage")); + assertTrue(messageTypes.contains("testoneofs.SampleMessage")); + + DynamicMessage.Builder sampleMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SampleMessage"); + Descriptors.Descriptor sampleMsgDescriptor = sampleMsgBuilder.getDescriptorForType(); + assertNotNull(sampleMsgDescriptor); + + List fields = sampleMsgDescriptor.getFields(); + assertEquals(2, fields.size()); + DynamicMessage sampleMsg = sampleMsgBuilder + .setField(sampleMsgDescriptor.findFieldByName("name"), "Bob") + .build(); + assertEquals("{\n" + " \"name\": \"Bob\"\n" + "}", dynamicMsgToJson(sampleMsgDescriptor, sampleMsg.toByteArray())); + + DynamicMessage.Builder subMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SubMessage"); + Descriptors.Descriptor subMsgDescriptor = subMsgBuilder.getDescriptorForType(); + DynamicMessage subMsg = subMsgBuilder + .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "Alice") + .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "John") + .build(); + + DynamicMessage sampleMsgWithOneOfSubMessage = sampleMsgBuilder.setField(sampleMsgDescriptor.findFieldByName("subMessage"), subMsg).build(); + assertEquals("{\n" + " \"subMessage\": {\n" + " \"name\": [\"Alice\", \"John\"]\n" + " }\n" + "}", + dynamicMsgToJson(sampleMsgDescriptor, sampleMsgWithOneOfSubMessage.toByteArray())); + } + + private DeviceProfile testSaveDeviceProfileWithProtoPayloadType(String schema) throws Exception { + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema); + MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration); + DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile foundDeviceProfile = doGet("/api/deviceProfile/"+savedDeviceProfile.getId().getId().toString(), DeviceProfile.class); + Assert.assertEquals(savedDeviceProfile.getName(), foundDeviceProfile.getName()); + return savedDeviceProfile; + } + + private void testSaveDeviceProfileWithInvalidProtoSchema(String schema, String errorMsg) throws Exception { + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema); + MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration); + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration); + doPost("/api/deviceProfile", deviceProfile).andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(errorMsg))); + } + + private DynamicSchema getDynamicSchema(String schema) throws Exception { + DeviceProfile deviceProfile = testSaveDeviceProfileWithProtoPayloadType(schema); + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration); + MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttDeviceProfileTransportConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + ProtoFileElement protoFile = protoTransportPayloadConfiguration.getTransportProtoSchema(schema); + return protoTransportPayloadConfiguration.getDynamicSchema(protoFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + } + + private String dynamicMsgToJson(Descriptors.Descriptor descriptor, byte[] payload) throws InvalidProtocolBufferException { + DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, payload); + return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage); + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java index a4d615ae2b..4f0d1e201b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java @@ -23,7 +23,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -285,7 +286,9 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController TenantProfile tenantProfile = new TenantProfile(); tenantProfile.setName(name); tenantProfile.setDescription(name + " Test"); - tenantProfile.setProfileData(new TenantProfileData()); + TenantProfileData tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + tenantProfile.setProfileData(tenantProfileData); tenantProfile.setDefault(false); tenantProfile.setIsolatedTbCore(false); tenantProfile.setIsolatedTbRuleEngine(false); diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index 64f9b8fddf..bc2c36b501 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -27,7 +27,7 @@ import java.util.Arrays; @RunWith(ClasspathSuite.class) @ClasspathSuite.ClassnameFilters({ // "org.thingsboard.server.controller.sql.WebsocketApiSqlTest", -// "org.thingsboard.server.controller.sql.EntityQueryControllerSqlTest", +// "org.thingsboard.server.controller.sql.TenantProfileControllerSqlTest", "org.thingsboard.server.controller.sql.*Test", }) public class ControllerSqlTestSuite { diff --git a/application/src/test/java/org/thingsboard/server/mqtt/AbstractMqttIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/AbstractMqttIntegrationTest.java index 337904718c..a1e4bc0e63 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/AbstractMqttIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/AbstractMqttIntegrationTest.java @@ -26,14 +26,22 @@ import org.junit.Assert; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.device.profile.AllowCreateNewDevicesDeviceProfileProvisionConfiguration; +import org.thingsboard.server.common.data.device.profile.CheckPreProvisionedDevicesDeviceProfileProvisionConfiguration; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.device.profile.DeviceProfileProvisionConfiguration; +import org.thingsboard.server.common.data.device.profile.DisabledDeviceProfileProvisionConfiguration; +import org.thingsboard.server.common.data.device.profile.JsonTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.controller.AbstractControllerTest; @@ -42,7 +50,6 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -55,6 +62,48 @@ public abstract class AbstractMqttIntegrationTest extends AbstractControllerTest private static final AtomicInteger atomicInteger = new AtomicInteger(2); + public static final String DEVICE_TELEMETRY_PROTO_SCHEMA = "syntax =\"proto3\";\n" + + "\n" + + "package test;\n" + + "\n" + + "message PostTelemetry {\n" + + " string key1 = 1;\n" + + " bool key2 = 2;\n" + + " double key3 = 3;\n" + + " int32 key4 = 4;\n" + + " JsonObject key5 = 5;\n" + + "\n" + + " message JsonObject {\n" + + " int32 someNumber = 6;\n" + + " repeated int32 someArray = 7;\n" + + " NestedJsonObject someNestedObject = 8;\n" + + " message NestedJsonObject {\n" + + " string key = 9;\n" + + " }\n" + + " }\n" + + "}"; + + public static final String DEVICE_ATTRIBUTES_PROTO_SCHEMA = "syntax =\"proto3\";\n" + + "\n" + + "package test;\n" + + "\n" + + "message PostAttributes {\n" + + " string key1 = 1;\n" + + " bool key2 = 2;\n" + + " double key3 = 3;\n" + + " int32 key4 = 4;\n" + + " JsonObject key5 = 5;\n" + + "\n" + + " message JsonObject {\n" + + " int32 someNumber = 6;\n" + + " repeated int32 someArray = 7;\n" + + " NestedJsonObject someNestedObject = 8;\n" + + " message NestedJsonObject {\n" + + " string key = 9;\n" + + " }\n" + + " }\n" + + "}"; + protected Tenant savedTenant; protected User tenantAdmin; @@ -64,7 +113,22 @@ public abstract class AbstractMqttIntegrationTest extends AbstractControllerTest protected Device savedGateway; protected String gatewayAccessToken; - protected void processBeforeTest(String deviceName, String gatewayName, TransportPayloadType payloadType, String telemetryTopic, String attributesTopic) throws Exception { + protected DeviceProfile deviceProfile; + + protected void processBeforeTest (String deviceName, String gatewayName, TransportPayloadType payloadType, String telemetryTopic, String attributesTopic) throws Exception { + this.processBeforeTest(deviceName, gatewayName, payloadType, telemetryTopic, attributesTopic, null, null, DeviceProfileProvisionType.DISABLED, null, null); + } + + protected void processBeforeTest(String deviceName, + String gatewayName, + TransportPayloadType payloadType, + String telemetryTopic, + String attributesTopic, + String telemetryProtoSchema, + String attributesProtoSchema, + DeviceProfileProvisionType provisionType, + String provisionKey, String provisionSecret + ) throws Exception { loginSysAdmin(); Tenant tenant = new Tenant(); @@ -93,12 +157,12 @@ public abstract class AbstractMqttIntegrationTest extends AbstractControllerTest gateway.setAdditionalInfo(additionalInfo); if (payloadType != null) { - DeviceProfile mqttDeviceProfile = createMqttDeviceProfile(payloadType, telemetryTopic, attributesTopic); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", mqttDeviceProfile, DeviceProfile.class); - device.setType(savedDeviceProfile.getName()); - device.setDeviceProfileId(savedDeviceProfile.getId()); - gateway.setType(savedDeviceProfile.getName()); - gateway.setDeviceProfileId(savedDeviceProfile.getId()); + DeviceProfile mqttDeviceProfile = createMqttDeviceProfile(payloadType, telemetryTopic, attributesTopic, telemetryProtoSchema, attributesProtoSchema, provisionType, provisionKey, provisionSecret); + deviceProfile = doPost("/api/deviceProfile", mqttDeviceProfile, DeviceProfile.class); + device.setType(deviceProfile.getName()); + device.setDeviceProfileId(deviceProfile.getId()); + gateway.setType(deviceProfile.getName()); + gateway.setDeviceProfileId(deviceProfile.getId()); } savedDevice = doPost("/api/device", device, Device.class); @@ -183,23 +247,58 @@ public abstract class AbstractMqttIntegrationTest extends AbstractControllerTest return keyValueProtoBuilder.build(); } - protected DeviceProfile createMqttDeviceProfile(TransportPayloadType transportPayloadType, String telemetryTopic, String attributesTopic) { + protected DeviceProfile createMqttDeviceProfile(TransportPayloadType transportPayloadType, + String telemetryTopic, String attributesTopic, + String telemetryProtoSchema, String attributesProtoSchema, + DeviceProfileProvisionType provisionType, + String provisionKey, String provisionSecret) { DeviceProfile deviceProfile = new DeviceProfile(); deviceProfile.setName(transportPayloadType.name()); deviceProfile.setType(DeviceProfileType.DEFAULT); deviceProfile.setTransportType(DeviceTransportType.MQTT); + deviceProfile.setProvisionType(provisionType); + deviceProfile.setProvisionDeviceKey(provisionKey); deviceProfile.setDescription(transportPayloadType.name() + " Test"); DeviceProfileData deviceProfileData = new DeviceProfileData(); DefaultDeviceProfileConfiguration configuration = new DefaultDeviceProfileConfiguration(); - MqttDeviceProfileTransportConfiguration transportConfiguration = new MqttDeviceProfileTransportConfiguration(); - transportConfiguration.setTransportPayloadType(transportPayloadType); + MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = new MqttDeviceProfileTransportConfiguration(); if (!StringUtils.isEmpty(telemetryTopic)) { - transportConfiguration.setDeviceTelemetryTopic(telemetryTopic); + mqttDeviceProfileTransportConfiguration.setDeviceTelemetryTopic(telemetryTopic); } if (!StringUtils.isEmpty(attributesTopic)) { - transportConfiguration.setDeviceAttributesTopic(attributesTopic); + mqttDeviceProfileTransportConfiguration.setDeviceAttributesTopic(attributesTopic); + } + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration; + if (TransportPayloadType.JSON.equals(transportPayloadType)) { + transportPayloadTypeConfiguration = new JsonTransportPayloadConfiguration(); + } else { + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = new ProtoTransportPayloadConfiguration(); + if (StringUtils.isEmpty(telemetryProtoSchema)) { + telemetryProtoSchema = DEVICE_TELEMETRY_PROTO_SCHEMA; + } + if (StringUtils.isEmpty(attributesProtoSchema)) { + attributesProtoSchema = DEVICE_ATTRIBUTES_PROTO_SCHEMA; + } + protoTransportPayloadConfiguration.setDeviceTelemetryProtoSchema(telemetryProtoSchema); + protoTransportPayloadConfiguration.setDeviceAttributesProtoSchema(attributesProtoSchema); + transportPayloadTypeConfiguration = protoTransportPayloadConfiguration; + } + mqttDeviceProfileTransportConfiguration.setTransportPayloadTypeConfiguration(transportPayloadTypeConfiguration); + deviceProfileData.setTransportConfiguration(mqttDeviceProfileTransportConfiguration); + DeviceProfileProvisionConfiguration provisionConfiguration; + switch (provisionType) { + case ALLOW_CREATE_NEW_DEVICES: + provisionConfiguration = new AllowCreateNewDevicesDeviceProfileProvisionConfiguration(provisionSecret); + break; + case CHECK_PRE_PROVISIONED_DEVICES: + provisionConfiguration = new CheckPreProvisionedDevicesDeviceProfileProvisionConfiguration(provisionSecret); + break; + case DISABLED: + default: + provisionConfiguration = new DisabledDeviceProfileProvisionConfiguration(provisionSecret); + break; } - deviceProfileData.setTransportConfiguration(transportConfiguration); + deviceProfileData.setProvisionConfiguration(provisionConfiguration); deviceProfileData.setConfiguration(configuration); deviceProfile.setProfileData(deviceProfileData); deviceProfile.setDefault(false); diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java index 095a5c3e7a..f71e8f37a2 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java @@ -31,7 +31,8 @@ import java.util.Arrays; "org.thingsboard.server.mqtt.telemetry.attributes.sql.*Test", "org.thingsboard.server.mqtt.attributes.updates.sql.*Test", "org.thingsboard.server.mqtt.attributes.request.sql.*Test", - "org.thingsboard.server.mqtt.claim.sql.*Test" + "org.thingsboard.server.mqtt.claim.sql.*Test", + "org.thingsboard.server.mqtt.provision.sql.*Test" }) public class MqttSqlTestSuite { diff --git a/application/src/test/java/org/thingsboard/server/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java index 32488c8eb0..d1b0c482f8 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java @@ -56,7 +56,6 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt return tsKvProtoList; } - protected TransportProtos.TsKvProto getTsKvProto(String key, String value, TransportProtos.KeyValueType keyValueType) { TransportProtos.TsKvProto.Builder tsKvProtoBuilder = TransportProtos.TsKvProto.newBuilder(); TransportProtos.KeyValueProto keyValueProto = getKeyValueProto(key, value, keyValueType); diff --git a/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestIntegrationTest.java index 9e4529caf2..51d38e6312 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestIntegrationTest.java @@ -18,9 +18,7 @@ package org.thingsboard.server.mqtt.attributes.request; import com.google.protobuf.InvalidProtocolBufferException; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; -import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; -import org.eclipse.paho.client.mqttv3.MqttCallback; import org.eclipse.paho.client.mqttv3.MqttException; import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; @@ -36,9 +34,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @Slf4j diff --git a/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestJsonIntegrationTest.java index 4b824ea0b5..83ef04f65e 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestJsonIntegrationTest.java @@ -18,14 +18,9 @@ package org.thingsboard.server.mqtt.attributes.request; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.TransportPayloadType; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - @Slf4j public abstract class AbstractMqttAttributesRequestJsonIntegrationTest extends AbstractMqttAttributesRequestIntegrationTest { diff --git a/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestProtoIntegrationTest.java index 96cc88fa2f..9e112c98be 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/AbstractMqttAttributesRequestProtoIntegrationTest.java @@ -15,18 +15,26 @@ */ package org.thingsboard.server.mqtt.attributes.request; +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.eclipse.paho.client.mqttv3.MqttException; import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.TransportPayloadType; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.MqttTopics; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; @@ -38,16 +46,33 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @Slf4j public abstract class AbstractMqttAttributesRequestProtoIntegrationTest extends AbstractMqttAttributesRequestIntegrationTest { - @Before - public void beforeTest() throws Exception { - processBeforeTest("Test Request attribute values from the server proto", "Gateway Test Request attribute values from the server proto", TransportPayloadType.PROTOBUF, null, null); - } + public static final String ATTRIBUTES_SCHEMA_STR = "syntax =\"proto3\";\n" + + "\n" + + "package test;\n" + + "\n" + + "message PostAttributes {\n" + + " string attribute1 = 1;\n" + + " bool attribute2 = 2;\n" + + " double attribute3 = 3;\n" + + " int32 attribute4 = 4;\n" + + " JsonObject attribute5 = 5;\n" + + "\n" + + " message JsonObject {\n" + + " int32 someNumber = 6;\n" + + " repeated int32 someArray = 7;\n" + + " NestedJsonObject someNestedObject = 8;\n" + + " message NestedJsonObject {\n" + + " string key = 9;\n" + + " }\n" + + " }\n" + + "}"; @After public void afterTest() throws Exception { @@ -56,21 +81,55 @@ public abstract class AbstractMqttAttributesRequestProtoIntegrationTest extends @Test public void testRequestAttributesValuesFromTheServer() throws Exception { + super.processBeforeTest("Test Request attribute values from the server proto", "Gateway Test Request attribute values from the server proto", + TransportPayloadType.PROTOBUF, null, null, null, ATTRIBUTES_SCHEMA_STR, DeviceProfileProvisionType.DISABLED, null, null); processTestRequestAttributesValuesFromTheServer(); } - @Test public void testRequestAttributesValuesFromTheServerGateway() throws Exception { + super.processBeforeTest("Test Request attribute values from the server proto", "Gateway Test Request attribute values from the server proto", TransportPayloadType.PROTOBUF, null, null); processTestGatewayRequestAttributesValuesFromTheServer(); } protected void postAttributesAndSubscribeToTopic(Device savedDevice, MqttAsyncClient client) throws Exception { doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - String keys = "attribute1,attribute2,attribute3,attribute4,attribute5"; - List expectedKeys = Arrays.asList(keys.split(",")); - TransportProtos.PostAttributeMsg postAttributeMsg = getPostAttributeMsg(expectedKeys); - byte[] payload = postAttributeMsg.toByteArray(); + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration); + MqttDeviceProfileTransportConfiguration mqttTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); + DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + + DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); + Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); + assertNotNull(nestedJsonObjectBuilderDescriptor); + DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build(); + + DynamicMessage.Builder jsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject"); + Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType(); + assertNotNull(jsonObjectBuilderDescriptor); + DynamicMessage jsonObject = jsonObjectBuilder + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3) + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject) + .build(); + + DynamicMessage.Builder postAttributesBuilder = attributesSchema.newMessageBuilder("PostAttributes"); + Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType(); + assertNotNull(postAttributesMsgDescriptor); + DynamicMessage postAttributesMsg = postAttributesBuilder + .setField(postAttributesMsgDescriptor.findFieldByName("attribute1"), "value1") + .setField(postAttributesMsgDescriptor.findFieldByName("attribute2"), true) + .setField(postAttributesMsgDescriptor.findFieldByName("attribute3"), 42.0) + .setField(postAttributesMsgDescriptor.findFieldByName("attribute4"), 73) + .setField(postAttributesMsgDescriptor.findFieldByName("attribute5"), jsonObject) + .build(); + byte[] payload = postAttributesMsg.toByteArray(); client.publish(MqttTopics.DEVICE_ATTRIBUTES_TOPIC, new MqttMessage(payload)); client.subscribe(MqttTopics.DEVICE_ATTRIBUTES_RESPONSES_TOPIC, MqttQoS.AT_MOST_ONCE.value()); } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/sql/MqttAttributesRequestJsonSqlIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/sql/MqttAttributesRequestJsonSqlIntegrationTest.java index d3750d49a3..146972bade 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/sql/MqttAttributesRequestJsonSqlIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/sql/MqttAttributesRequestJsonSqlIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.mqtt.attributes.request.sql; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.mqtt.attributes.request.AbstractMqttAttributesRequestIntegrationTest; import org.thingsboard.server.mqtt.attributes.request.AbstractMqttAttributesRequestJsonIntegrationTest; @DaoSqlTest diff --git a/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/sql/MqttAttributesRequestProtoSqlIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/sql/MqttAttributesRequestProtoSqlIntegrationTest.java index f52e79b2ab..52ec27f6f8 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/sql/MqttAttributesRequestProtoSqlIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/attributes/request/sql/MqttAttributesRequestProtoSqlIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.mqtt.attributes.request.sql; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.mqtt.attributes.request.AbstractMqttAttributesRequestJsonIntegrationTest; import org.thingsboard.server.mqtt.attributes.request.AbstractMqttAttributesRequestProtoIntegrationTest; @DaoSqlTest diff --git a/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesIntegrationTest.java index d2febdf357..b91739790c 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesIntegrationTest.java @@ -18,11 +18,7 @@ package org.thingsboard.server.mqtt.attributes.updates; import com.google.protobuf.InvalidProtocolBufferException; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; -import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; -import org.eclipse.paho.client.mqttv3.MqttCallback; -import org.eclipse.paho.client.mqttv3.MqttException; -import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -33,12 +29,10 @@ import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.mqtt.attributes.AbstractMqttAttributesIntegrationTest; import java.nio.charset.StandardCharsets; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @Slf4j diff --git a/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesJsonIntegrationTest.java index 58379e4016..e41ff83291 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesJsonIntegrationTest.java @@ -21,11 +21,6 @@ import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.TransportPayloadType; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - @Slf4j public abstract class AbstractMqttAttributesUpdatesJsonIntegrationTest extends AbstractMqttAttributesUpdatesIntegrationTest { diff --git a/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesProtoIntegrationTest.java index faf8e1ce4d..213540699b 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/AbstractMqttAttributesUpdatesProtoIntegrationTest.java @@ -21,11 +21,9 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.TransportPayloadType; -import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.stream.Collectors; diff --git a/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/sql/MqttAttributesUpdatesSqlJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/sql/MqttAttributesUpdatesSqlJsonIntegrationTest.java index a8fd4687f7..1841340835 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/sql/MqttAttributesUpdatesSqlJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/sql/MqttAttributesUpdatesSqlJsonIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.mqtt.attributes.updates.sql; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.mqtt.attributes.updates.AbstractMqttAttributesUpdatesIntegrationTest; import org.thingsboard.server.mqtt.attributes.updates.AbstractMqttAttributesUpdatesJsonIntegrationTest; @DaoSqlTest diff --git a/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/sql/MqttAttributesUpdatesSqlProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/sql/MqttAttributesUpdatesSqlProtoIntegrationTest.java index 723e5e3dcd..7ff542a4ec 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/sql/MqttAttributesUpdatesSqlProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/attributes/updates/sql/MqttAttributesUpdatesSqlProtoIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.mqtt.attributes.updates.sql; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.mqtt.attributes.updates.AbstractMqttAttributesUpdatesJsonIntegrationTest; import org.thingsboard.server.mqtt.attributes.updates.AbstractMqttAttributesUpdatesProtoIntegrationTest; @DaoSqlTest diff --git a/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimDeviceTest.java b/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimDeviceTest.java index dd16c17be2..5a1cbd6356 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimDeviceTest.java @@ -20,7 +20,6 @@ import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.ClaimRequest; import org.thingsboard.server.common.data.Customer; diff --git a/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimJsonDeviceTest.java b/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimJsonDeviceTest.java index f55cfa57c8..49aa6c995b 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimJsonDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimJsonDeviceTest.java @@ -18,7 +18,6 @@ package org.thingsboard.server.mqtt.claim; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.TransportPayloadType; diff --git a/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimProtoDeviceTest.java b/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimProtoDeviceTest.java index d371c09f37..be0cfc7c81 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimProtoDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/claim/AbstractMqttClaimProtoDeviceTest.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.gen.transport.TransportApiProtos; diff --git a/application/src/test/java/org/thingsboard/server/mqtt/claim/sql/MqttClaimDeviceJsonSqlTest.java b/application/src/test/java/org/thingsboard/server/mqtt/claim/sql/MqttClaimDeviceJsonSqlTest.java index da794288f4..543d805721 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/claim/sql/MqttClaimDeviceJsonSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/claim/sql/MqttClaimDeviceJsonSqlTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.mqtt.claim.sql; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.mqtt.claim.AbstractMqttClaimDeviceTest; import org.thingsboard.server.mqtt.claim.AbstractMqttClaimJsonDeviceTest; @DaoSqlTest diff --git a/application/src/test/java/org/thingsboard/server/mqtt/claim/sql/MqttClaimDeviceProtoSqlTest.java b/application/src/test/java/org/thingsboard/server/mqtt/claim/sql/MqttClaimDeviceProtoSqlTest.java index a63978e4de..e5d866d6e2 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/claim/sql/MqttClaimDeviceProtoSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/claim/sql/MqttClaimDeviceProtoSqlTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.mqtt.claim.sql; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.mqtt.claim.AbstractMqttClaimJsonDeviceTest; import org.thingsboard.server.mqtt.claim.AbstractMqttClaimProtoDeviceTest; @DaoSqlTest diff --git a/application/src/test/java/org/thingsboard/server/mqtt/provision/AbstractMqttProvisionJsonDeviceTest.java b/application/src/test/java/org/thingsboard/server/mqtt/provision/AbstractMqttProvisionJsonDeviceTest.java new file mode 100644 index 0000000000..6f96ae1034 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/mqtt/provision/AbstractMqttProvisionJsonDeviceTest.java @@ -0,0 +1,274 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.mqtt.provision; + +import com.google.gson.JsonObject; +import io.netty.handler.codec.mqtt.MqttQoS; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; +import org.eclipse.paho.client.mqttv3.MqttAsyncClient; +import org.eclipse.paho.client.mqttv3.MqttCallback; +import org.eclipse.paho.client.mqttv3.MqttMessage; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.TransportPayloadType; +import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; +import org.thingsboard.server.common.data.device.profile.MqttTopics; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.msg.EncryptionUtil; +import org.thingsboard.server.common.transport.util.JsonUtils; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus; +import org.thingsboard.server.dao.util.mapping.JacksonUtil; +import org.thingsboard.server.mqtt.AbstractMqttIntegrationTest; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +@Slf4j +public abstract class AbstractMqttProvisionJsonDeviceTest extends AbstractMqttIntegrationTest { + + @Autowired + DeviceCredentialsService deviceCredentialsService; + + @Autowired + DeviceService deviceService; + + @After + public void afterTest() throws Exception { + super.processAfterTest(); + } + + @Test + public void testProvisioningDisabledDevice() throws Exception { + processTestProvisioningDisabledDevice(); + } + + @Test + public void testProvisioningCheckPreProvisionedDevice() throws Exception { + processTestProvisioningCheckPreProvisionedDevice(); + } + + @Test + public void testProvisioningCreateNewDeviceWithoutCredentials() throws Exception { + processTestProvisioningCreateNewDeviceWithoutCredentials(); + } + + @Test + public void testProvisioningCreateNewDeviceWithAccessToken() throws Exception { + processTestProvisioningCreateNewDeviceWithAccessToken(); + } + + @Test + public void testProvisioningCreateNewDeviceWithCert() throws Exception { + processTestProvisioningCreateNewDeviceWithCert(); + } + + @Test + public void testProvisioningCreateNewDeviceWithMqttBasic() throws Exception { + processTestProvisioningCreateNewDeviceWithMqttBasic(); + } + + @Test + public void testProvisioningWithBadKeyDevice() throws Exception { + processTestProvisioningWithBadKeyDevice(); + } + + + protected void processTestProvisioningDisabledDevice() throws Exception { + super.processBeforeTest("Test Provision device", "Test Provision gateway", TransportPayloadType.JSON, null, null, null, null, DeviceProfileProvisionType.DISABLED, null, null); + byte[] result = createMqttClientAndPublish().getPayloadBytes(); + JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + Assert.assertEquals("Provision data was not found!", response.get("errorMsg").getAsString()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").getAsString()); + } + + + protected void processTestProvisioningCreateNewDeviceWithoutCredentials() throws Exception { + super.processBeforeTest("Test Provision device3", "Test Provision gateway", TransportPayloadType.JSON, null, null, null, null, DeviceProfileProvisionType.ALLOW_CREATE_NEW_DEVICES, "testProvisionKey", "testProvisionSecret"); + byte[] result = createMqttClientAndPublish().getPayloadBytes(); + JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + + Device createdDevice = deviceService.findDeviceByTenantIdAndName(savedTenant.getTenantId(), "Test Provision device"); + + Assert.assertNotNull(createdDevice); + + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(savedTenant.getTenantId(), createdDevice.getId()); + + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + } + + + protected void processTestProvisioningCreateNewDeviceWithAccessToken() throws Exception { + super.processBeforeTest("Test Provision device3", "Test Provision gateway", TransportPayloadType.JSON, null, null, null, null, DeviceProfileProvisionType.ALLOW_CREATE_NEW_DEVICES, "testProvisionKey", "testProvisionSecret"); + String requestCredentials = ",\"credentialsType\": \"ACCESS_TOKEN\",\"token\": \"test_token\""; + byte[] result = createMqttClientAndPublish(requestCredentials).getPayloadBytes(); + JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + + Device createdDevice = deviceService.findDeviceByTenantIdAndName(savedTenant.getTenantId(), "Test Provision device"); + + Assert.assertNotNull(createdDevice); + + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(savedTenant.getTenantId(), createdDevice.getId()); + + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), "ACCESS_TOKEN"); + Assert.assertEquals(deviceCredentials.getCredentialsId(), "test_token"); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + } + + + protected void processTestProvisioningCreateNewDeviceWithCert() throws Exception { + super.processBeforeTest("Test Provision device3", "Test Provision gateway", TransportPayloadType.JSON, null, null, null, null, DeviceProfileProvisionType.ALLOW_CREATE_NEW_DEVICES, "testProvisionKey", "testProvisionSecret"); + String requestCredentials = ",\"credentialsType\": \"X509_CERTIFICATE\",\"hash\": \"testHash\""; + byte[] result = createMqttClientAndPublish(requestCredentials).getPayloadBytes(); + JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + + Device createdDevice = deviceService.findDeviceByTenantIdAndName(savedTenant.getTenantId(), "Test Provision device"); + + Assert.assertNotNull(createdDevice); + + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(savedTenant.getTenantId(), createdDevice.getId()); + + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), "X509_CERTIFICATE"); + + String cert = EncryptionUtil.trimNewLines(deviceCredentials.getCredentialsValue()); + String sha3Hash = EncryptionUtil.getSha3Hash(cert); + + Assert.assertEquals(deviceCredentials.getCredentialsId(), sha3Hash); + + Assert.assertEquals(deviceCredentials.getCredentialsValue(), "testHash"); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + } + + + protected void processTestProvisioningCreateNewDeviceWithMqttBasic() throws Exception { + super.processBeforeTest("Test Provision device3", "Test Provision gateway", TransportPayloadType.JSON, null, null, null, null, DeviceProfileProvisionType.ALLOW_CREATE_NEW_DEVICES, "testProvisionKey", "testProvisionSecret"); + String requestCredentials = ",\"credentialsType\": \"MQTT_BASIC\",\"clientId\": \"test_clientId\",\"username\": \"test_username\",\"password\": \"test_password\""; + byte[] result = createMqttClientAndPublish(requestCredentials).getPayloadBytes(); + JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + + Device createdDevice = deviceService.findDeviceByTenantIdAndName(savedTenant.getTenantId(), "Test Provision device"); + + Assert.assertNotNull(createdDevice); + + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(savedTenant.getTenantId(), createdDevice.getId()); + + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), "MQTT_BASIC"); + Assert.assertEquals(deviceCredentials.getCredentialsId(), EncryptionUtil.getSha3Hash("|", "test_clientId", "test_username")); + + BasicMqttCredentials mqttCredentials = new BasicMqttCredentials(); + mqttCredentials.setClientId("test_clientId"); + mqttCredentials.setUserName("test_username"); + mqttCredentials.setPassword("test_password"); + + Assert.assertEquals(deviceCredentials.getCredentialsValue(), JacksonUtil.toString(mqttCredentials)); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + } + + protected void processTestProvisioningCheckPreProvisionedDevice() throws Exception { + super.processBeforeTest("Test Provision device", "Test Provision gateway", TransportPayloadType.JSON, null, null, null, null, DeviceProfileProvisionType.CHECK_PRE_PROVISIONED_DEVICES, "testProvisionKey", "testProvisionSecret"); + byte[] result = createMqttClientAndPublish().getPayloadBytes(); + JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(savedTenant.getTenantId(), savedDevice.getId()); + + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + } + + protected void processTestProvisioningWithBadKeyDevice() throws Exception { + super.processBeforeTest("Test Provision device", "Test Provision gateway", TransportPayloadType.JSON, null, null, null, null, DeviceProfileProvisionType.CHECK_PRE_PROVISIONED_DEVICES, "testProvisionKeyOrig", "testProvisionSecret"); + byte[] result = createMqttClientAndPublish().getPayloadBytes(); + JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + Assert.assertEquals("Provision data was not found!", response.get("errorMsg").getAsString()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").getAsString()); + } + + protected TestMqttCallback createMqttClientAndPublish() throws Exception { + return createMqttClientAndPublish(""); + } + + protected TestMqttCallback createMqttClientAndPublish(String deviceCredentials) throws Exception { + String provisionRequestMsg = createTestProvisionMessage(deviceCredentials); + MqttAsyncClient client = getMqttAsyncClient("provision"); + TestMqttCallback onProvisionCallback = getTestMqttCallback(); + client.setCallback(onProvisionCallback); + client.subscribe(MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC, MqttQoS.AT_MOST_ONCE.value()); + Thread.sleep(2000); + client.publish(MqttTopics.DEVICE_PROVISION_REQUEST_TOPIC, new MqttMessage(provisionRequestMsg.getBytes())); + onProvisionCallback.getLatch().await(3, TimeUnit.SECONDS); + return onProvisionCallback; + } + + + protected TestMqttCallback getTestMqttCallback() { + CountDownLatch latch = new CountDownLatch(1); + return new TestMqttCallback(latch); + } + + + protected static class TestMqttCallback implements MqttCallback { + + private final CountDownLatch latch; + private Integer qoS; + private byte[] payloadBytes; + + TestMqttCallback(CountDownLatch latch) { + this.latch = latch; + } + + public int getQoS() { + return qoS; + } + + public byte[] getPayloadBytes() { + return payloadBytes; + } + + public CountDownLatch getLatch() { + return latch; + } + + @Override + public void connectionLost(Throwable throwable) { + } + + @Override + public void messageArrived(String requestTopic, MqttMessage mqttMessage) throws Exception { + qoS = mqttMessage.getQos(); + payloadBytes = mqttMessage.getPayload(); + latch.countDown(); + } + + @Override + public void deliveryComplete(IMqttDeliveryToken iMqttDeliveryToken) { + + } + } + + protected String createTestProvisionMessage(String deviceCredentials) { + return "{\"deviceName\":\"Test Provision device\",\"provisionDeviceKey\":\"testProvisionKey\", \"provisionDeviceSecret\":\"testProvisionSecret\"" + deviceCredentials + "}"; + } +} diff --git a/application/src/test/java/org/thingsboard/server/mqtt/provision/AbstractMqttProvisionProtoDeviceTest.java b/application/src/test/java/org/thingsboard/server/mqtt/provision/AbstractMqttProvisionProtoDeviceTest.java new file mode 100644 index 0000000000..9f74946cad --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/mqtt/provision/AbstractMqttProvisionProtoDeviceTest.java @@ -0,0 +1,294 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.mqtt.provision; + +import io.netty.handler.codec.mqtt.MqttQoS; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; +import org.eclipse.paho.client.mqttv3.MqttAsyncClient; +import org.eclipse.paho.client.mqttv3.MqttCallback; +import org.eclipse.paho.client.mqttv3.MqttMessage; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.TransportPayloadType; +import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; +import org.thingsboard.server.common.data.device.profile.MqttTopics; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import org.thingsboard.server.common.msg.EncryptionUtil; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus; +import org.thingsboard.server.dao.util.mapping.JacksonUtil; +import org.thingsboard.server.gen.transport.TransportProtos.CredentialsDataProto; +import org.thingsboard.server.gen.transport.TransportProtos.CredentialsType; +import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceCredentialsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateBasicMqttCredRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.mqtt.AbstractMqttIntegrationTest; + +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +@Slf4j +public abstract class AbstractMqttProvisionProtoDeviceTest extends AbstractMqttIntegrationTest { + + @Autowired + DeviceCredentialsService deviceCredentialsService; + + @Autowired + DeviceService deviceService; + + @After + public void afterTest() throws Exception { + super.processAfterTest(); + } + + @Test + public void testProvisioningDisabledDevice() throws Exception { + processTestProvisioningDisabledDevice(); + } + + @Test + public void testProvisioningCheckPreProvisionedDevice() throws Exception { + processTestProvisioningCheckPreProvisionedDevice(); + } + + @Test + public void testProvisioningCreateNewDeviceWithoutCredentials() throws Exception { + processTestProvisioningCreateNewDeviceWithoutCredentials(); + } + + @Test + public void testProvisioningCreateNewDeviceWithAccessToken() throws Exception { + processTestProvisioningCreateNewDeviceWithAccessToken(); + } + + @Test + public void testProvisioningCreateNewDeviceWithCert() throws Exception { + processTestProvisioningCreateNewDeviceWithCert(); + } + + @Test + public void testProvisioningCreateNewDeviceWithMqttBasic() throws Exception { + processTestProvisioningCreateNewDeviceWithMqttBasic(); + } + + @Test + public void testProvisioningWithBadKeyDevice() throws Exception { + processTestProvisioningWithBadKeyDevice(); + } + + + protected void processTestProvisioningDisabledDevice() throws Exception { + super.processBeforeTest("Test Provision device", "Test Provision gateway", TransportPayloadType.PROTOBUF, null, null, null, null, DeviceProfileProvisionType.DISABLED, null, null); + ProvisionDeviceResponseMsg result = ProvisionDeviceResponseMsg.parseFrom(createMqttClientAndPublish().getPayloadBytes()); + Assert.assertNotNull(result); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), result.getStatus().toString()); + } + + protected void processTestProvisioningCreateNewDeviceWithoutCredentials() throws Exception { + super.processBeforeTest("Test Provision device3", "Test Provision gateway", TransportPayloadType.PROTOBUF, null, null, null, null, DeviceProfileProvisionType.ALLOW_CREATE_NEW_DEVICES, "testProvisionKey", "testProvisionSecret"); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createMqttClientAndPublish().getPayloadBytes()); + + Device createdDevice = deviceService.findDeviceByTenantIdAndName(savedTenant.getTenantId(), "Test Provision device"); + + Assert.assertNotNull(createdDevice); + + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(savedTenant.getTenantId(), createdDevice.getId()); + + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString()); + } + + protected void processTestProvisioningCreateNewDeviceWithAccessToken() throws Exception { + super.processBeforeTest("Test Provision device3", "Test Provision gateway", TransportPayloadType.PROTOBUF, null, null,null, null, DeviceProfileProvisionType.ALLOW_CREATE_NEW_DEVICES, "testProvisionKey", "testProvisionSecret"); + CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder().setValidateDeviceTokenRequestMsg(ValidateDeviceTokenRequestMsg.newBuilder().setToken("test_token").build()).build(); + + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createMqttClientAndPublish(createTestsProvisionMessage(CredentialsType.ACCESS_TOKEN, requestCredentials)).getPayloadBytes()); + + Device createdDevice = deviceService.findDeviceByTenantIdAndName(savedTenant.getTenantId(), "Test Provision device"); + + Assert.assertNotNull(createdDevice); + + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(savedTenant.getTenantId(), createdDevice.getId()); + + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString()); + Assert.assertEquals(deviceCredentials.getCredentialsType(), DeviceCredentialsType.ACCESS_TOKEN); + Assert.assertEquals(deviceCredentials.getCredentialsId(), "test_token"); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString()); + } + + protected void processTestProvisioningCreateNewDeviceWithCert() throws Exception { + super.processBeforeTest("Test Provision device3", "Test Provision gateway", TransportPayloadType.PROTOBUF, null, null, null, null, DeviceProfileProvisionType.ALLOW_CREATE_NEW_DEVICES, "testProvisionKey", "testProvisionSecret"); + CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder().setValidateDeviceX509CertRequestMsg(ValidateDeviceX509CertRequestMsg.newBuilder().setHash("testHash").build()).build(); + + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createMqttClientAndPublish(createTestsProvisionMessage(CredentialsType.X509_CERTIFICATE, requestCredentials)).getPayloadBytes()); + + Device createdDevice = deviceService.findDeviceByTenantIdAndName(savedTenant.getTenantId(), "Test Provision device"); + + Assert.assertNotNull(createdDevice); + + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(savedTenant.getTenantId(), createdDevice.getId()); + + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString()); + Assert.assertEquals(deviceCredentials.getCredentialsType(), DeviceCredentialsType.X509_CERTIFICATE); + + String cert = EncryptionUtil.trimNewLines(deviceCredentials.getCredentialsValue()); + String sha3Hash = EncryptionUtil.getSha3Hash(cert); + + Assert.assertEquals(deviceCredentials.getCredentialsId(), sha3Hash); + + Assert.assertEquals(deviceCredentials.getCredentialsValue(), "testHash"); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString()); + } + + protected void processTestProvisioningCreateNewDeviceWithMqttBasic() throws Exception { + super.processBeforeTest("Test Provision device3", "Test Provision gateway", TransportPayloadType.PROTOBUF, null, null, null, null, DeviceProfileProvisionType.ALLOW_CREATE_NEW_DEVICES, "testProvisionKey", "testProvisionSecret"); + CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder().setValidateBasicMqttCredRequestMsg( + ValidateBasicMqttCredRequestMsg.newBuilder() + .setClientId("test_clientId") + .setUserName("test_username") + .setPassword("test_password") + .build() + ).build(); + + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createMqttClientAndPublish(createTestsProvisionMessage(CredentialsType.MQTT_BASIC, requestCredentials)).getPayloadBytes()); + + Device createdDevice = deviceService.findDeviceByTenantIdAndName(savedTenant.getTenantId(), "Test Provision device"); + + Assert.assertNotNull(createdDevice); + + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(savedTenant.getTenantId(), createdDevice.getId()); + + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString()); + Assert.assertEquals(deviceCredentials.getCredentialsType(), DeviceCredentialsType.MQTT_BASIC); + Assert.assertEquals(deviceCredentials.getCredentialsId(), EncryptionUtil.getSha3Hash("|", "test_clientId", "test_username")); + + BasicMqttCredentials mqttCredentials = new BasicMqttCredentials(); + mqttCredentials.setClientId("test_clientId"); + mqttCredentials.setUserName("test_username"); + mqttCredentials.setPassword("test_password"); + + Assert.assertEquals(deviceCredentials.getCredentialsValue(), JacksonUtil.toString(mqttCredentials)); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString()); + } + + protected void processTestProvisioningCheckPreProvisionedDevice() throws Exception { + super.processBeforeTest("Test Provision device", "Test Provision gateway", TransportPayloadType.PROTOBUF, null, null, null, null, DeviceProfileProvisionType.CHECK_PRE_PROVISIONED_DEVICES, "testProvisionKey", "testProvisionSecret"); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createMqttClientAndPublish().getPayloadBytes()); + + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(savedTenant.getTenantId(), savedDevice.getId()); + + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString()); + } + + protected void processTestProvisioningWithBadKeyDevice() throws Exception { + super.processBeforeTest("Test Provision device", "Test Provision gateway", TransportPayloadType.PROTOBUF, null, null, null, null, DeviceProfileProvisionType.CHECK_PRE_PROVISIONED_DEVICES, "testProvisionKeyOrig", "testProvisionSecret"); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createMqttClientAndPublish().getPayloadBytes()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.getStatus().toString()); + } + + protected TestMqttCallback createMqttClientAndPublish() throws Exception { + byte[] provisionRequestMsg = createTestProvisionMessage(); + return createMqttClientAndPublish(provisionRequestMsg); + } + + protected TestMqttCallback createMqttClientAndPublish(byte[] provisionRequestMsg) throws Exception { + MqttAsyncClient client = getMqttAsyncClient("provision"); + TestMqttCallback onProvisionCallback = getTestMqttCallback(); + client.setCallback(onProvisionCallback); + client.subscribe(MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC, MqttQoS.AT_MOST_ONCE.value()); + Thread.sleep(2000); + client.publish(MqttTopics.DEVICE_PROVISION_REQUEST_TOPIC, new MqttMessage(provisionRequestMsg)); + onProvisionCallback.getLatch().await(3, TimeUnit.SECONDS); + return onProvisionCallback; + } + + + protected TestMqttCallback getTestMqttCallback() { + CountDownLatch latch = new CountDownLatch(1); + return new TestMqttCallback(latch); + } + + + protected static class TestMqttCallback implements MqttCallback { + + private final CountDownLatch latch; + private Integer qoS; + private byte[] payloadBytes; + + TestMqttCallback(CountDownLatch latch) { + this.latch = latch; + } + + public int getQoS() { + return qoS; + } + + public byte[] getPayloadBytes() { + return payloadBytes; + } + + public CountDownLatch getLatch() { + return latch; + } + + @Override + public void connectionLost(Throwable throwable) { + } + + @Override + public void messageArrived(String requestTopic, MqttMessage mqttMessage) throws Exception { + qoS = mqttMessage.getQos(); + payloadBytes = mqttMessage.getPayload(); + latch.countDown(); + } + + @Override + public void deliveryComplete(IMqttDeliveryToken iMqttDeliveryToken) { + + } + } + + protected byte[] createTestsProvisionMessage(CredentialsType credentialsType, CredentialsDataProto credentialsData) throws Exception { + return ProvisionDeviceRequestMsg.newBuilder() + .setDeviceName("Test Provision device") + .setCredentialsType(credentialsType != null ? credentialsType : CredentialsType.ACCESS_TOKEN) + .setCredentialsDataProto(credentialsData != null ? credentialsData: CredentialsDataProto.newBuilder().build()) + .setProvisionDeviceCredentialsMsg( + ProvisionDeviceCredentialsMsg.newBuilder() + .setProvisionDeviceKey("testProvisionKey") + .setProvisionDeviceSecret("testProvisionSecret") + ).build() + .toByteArray(); + } + + + protected byte[] createTestProvisionMessage() throws Exception { + return createTestsProvisionMessage(null, null); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/mqtt/provision/sql/MqttProvisionDeviceJsonSqlTest.java b/application/src/test/java/org/thingsboard/server/mqtt/provision/sql/MqttProvisionDeviceJsonSqlTest.java new file mode 100644 index 0000000000..c9e6495fd3 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/mqtt/provision/sql/MqttProvisionDeviceJsonSqlTest.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.mqtt.provision.sql; + +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.mqtt.provision.AbstractMqttProvisionJsonDeviceTest; + +@DaoSqlTest +public class MqttProvisionDeviceJsonSqlTest extends AbstractMqttProvisionJsonDeviceTest { +} diff --git a/application/src/test/java/org/thingsboard/server/mqtt/provision/sql/MqttProvisionDeviceProtoSqlTest.java b/application/src/test/java/org/thingsboard/server/mqtt/provision/sql/MqttProvisionDeviceProtoSqlTest.java new file mode 100644 index 0000000000..7a8170d01e --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/mqtt/provision/sql/MqttProvisionDeviceProtoSqlTest.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.mqtt.provision.sql; + +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.mqtt.provision.AbstractMqttProvisionProtoDeviceTest; + +@DaoSqlTest +public class MqttProvisionDeviceProtoSqlTest extends AbstractMqttProvisionProtoDeviceTest { +} diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcDefaultIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcDefaultIntegrationTest.java index 23b93f427e..e3de80e813 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcDefaultIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcDefaultIntegrationTest.java @@ -15,49 +15,14 @@ */ package org.thingsboard.server.mqtt.rpc; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.protobuf.InvalidProtocolBufferException; -import com.nimbusds.jose.util.StandardCharset; import com.datastax.oss.driver.api.core.uuid.Uuids; -import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; -import org.eclipse.paho.client.mqttv3.MqttAsyncClient; -import org.eclipse.paho.client.mqttv3.MqttCallback; -import org.eclipse.paho.client.mqttv3.MqttConnectOptions; -import org.eclipse.paho.client.mqttv3.MqttException; -import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.DeviceProfileType; -import org.thingsboard.server.common.data.DeviceTransportType; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.TransportPayloadType; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; -import org.thingsboard.server.common.data.device.profile.DeviceProfileData; -import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.MqttTopics; -import org.thingsboard.server.common.data.security.Authority; -import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.controller.AbstractControllerTest; -import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.service.security.AccessValidator; -import java.util.Arrays; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; /** diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index e08f1665a4..208aa03a48 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.mqtt.rpc; -import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.protobuf.InvalidProtocolBufferException; import com.nimbusds.jose.util.StandardCharset; import io.netty.handler.codec.mqtt.MqttQoS; @@ -26,35 +24,18 @@ import org.apache.commons.lang3.StringUtils; import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.eclipse.paho.client.mqttv3.MqttCallback; -import org.eclipse.paho.client.mqttv3.MqttConnectOptions; import org.eclipse.paho.client.mqttv3.MqttException; import org.eclipse.paho.client.mqttv3.MqttMessage; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.DeviceProfileType; -import org.thingsboard.server.common.data.DeviceTransportType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TransportPayloadType; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; -import org.thingsboard.server.common.data.device.profile.DeviceProfileData; -import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttTopics; -import org.thingsboard.server.common.data.security.Authority; -import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.mqtt.AbstractMqttIntegrationTest; -import org.thingsboard.server.service.security.AccessValidator; import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcJsonIntegrationTest.java index d9ff14e1d2..2bfb0c1f2d 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcJsonIntegrationTest.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.TransportPayloadType; diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcProtoIntegrationTest.java index 759a5da912..062a074c0d 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcProtoIntegrationTest.java @@ -22,16 +22,12 @@ import org.eclipse.paho.client.mqttv3.MqttException; import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - @Slf4j public abstract class AbstractMqttServerSideRpcProtoIntegrationTest extends AbstractMqttServerSideRpcIntegrationTest { diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/AbstractMqttAttributesIntegrationTest.java index 5ac0746a43..0c8e963d1b 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/AbstractMqttAttributesIntegrationTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.mqtt.telemetry.attributes; +import com.fasterxml.jackson.core.JsonProcessingException; import lombok.extern.slf4j.Slf4j; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.junit.After; @@ -142,7 +143,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } - protected void assertAttributesValues(List> deviceValues, Set expectedKeySet) { + protected void assertAttributesValues(List> deviceValues, Set expectedKeySet) throws JsonProcessingException { for (Map map : deviceValues) { String key = (String) map.get("key"); Object value = map.get("value"); diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java index e9adf19359..1e14541d23 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java @@ -15,18 +15,24 @@ */ package org.thingsboard.server.mqtt.telemetry.attributes; +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.TransportPayloadType; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -35,11 +41,6 @@ public abstract class AbstractMqttAttributesProtoIntegrationTest extends Abstrac private static final String POST_DATA_ATTRIBUTES_TOPIC = "proto/attributes"; - @Before - public void beforeTest() throws Exception { - processBeforeTest("Test Post Attributes device", "Test Post Attributes gateway", TransportPayloadType.PROTOBUF, null, POST_DATA_ATTRIBUTES_TOPIC); - } - @After public void afterTest() throws Exception { processAfterTest(); @@ -47,13 +48,49 @@ public abstract class AbstractMqttAttributesProtoIntegrationTest extends Abstrac @Test public void testPushMqttAttributes() throws Exception { + super.processBeforeTest("Test Post Attributes device", "Test Post Attributes gateway", TransportPayloadType.PROTOBUF, null, POST_DATA_ATTRIBUTES_TOPIC); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4", "key5"); - TransportProtos.PostAttributeMsg msg = getPostAttributeMsg(expectedKeys); - processAttributesTest(POST_DATA_ATTRIBUTES_TOPIC, expectedKeys, msg.toByteArray()); + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration); + MqttDeviceProfileTransportConfiguration mqttTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA); + DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + + DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); + Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); + assertNotNull(nestedJsonObjectBuilderDescriptor); + DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build(); + + DynamicMessage.Builder jsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject"); + Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType(); + assertNotNull(jsonObjectBuilderDescriptor); + DynamicMessage jsonObject = jsonObjectBuilder + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3) + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject) + .build(); + + DynamicMessage.Builder postAttributesBuilder = attributesSchema.newMessageBuilder("PostAttributes"); + Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType(); + assertNotNull(postAttributesMsgDescriptor); + DynamicMessage postAttributesMsg = postAttributesBuilder + .setField(postAttributesMsgDescriptor.findFieldByName("key1"), "value1") + .setField(postAttributesMsgDescriptor.findFieldByName("key2"), true) + .setField(postAttributesMsgDescriptor.findFieldByName("key3"), 3.0) + .setField(postAttributesMsgDescriptor.findFieldByName("key4"), 4) + .setField(postAttributesMsgDescriptor.findFieldByName("key5"), jsonObject) + .build(); + processAttributesTest(POST_DATA_ATTRIBUTES_TOPIC, expectedKeys, postAttributesMsg.toByteArray()); } @Test public void testPushMqttAttributesGateway() throws Exception { + super.processBeforeTest("Test Post Attributes device", "Test Post Attributes gateway", TransportPayloadType.PROTOBUF, null, null); TransportApiProtos.GatewayAttributesMsg.Builder gatewayAttributesMsgProtoBuilder = TransportApiProtos.GatewayAttributesMsg.newBuilder(); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4", "key5"); String deviceName1 = "Device A"; diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/nosql/MqttAttributesNoSqlJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/nosql/MqttAttributesNoSqlJsonIntegrationTest.java index 68ab8cff62..53a6f4adde 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/nosql/MqttAttributesNoSqlJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/nosql/MqttAttributesNoSqlJsonIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.mqtt.telemetry.attributes.nosql; import org.thingsboard.server.dao.service.DaoNoSqlTest; -import org.thingsboard.server.mqtt.telemetry.attributes.AbstractMqttAttributesIntegrationTest; import org.thingsboard.server.mqtt.telemetry.attributes.AbstractMqttAttributesJsonIntegrationTest; @DaoNoSqlTest diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/nosql/MqttAttributesNoSqlProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/nosql/MqttAttributesNoSqlProtoIntegrationTest.java index d508fd76c5..23b6052272 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/nosql/MqttAttributesNoSqlProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/nosql/MqttAttributesNoSqlProtoIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.mqtt.telemetry.attributes.nosql; import org.thingsboard.server.dao.service.DaoNoSqlTest; -import org.thingsboard.server.mqtt.telemetry.attributes.AbstractMqttAttributesIntegrationTest; import org.thingsboard.server.mqtt.telemetry.attributes.AbstractMqttAttributesProtoIntegrationTest; @DaoNoSqlTest diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/sql/MqttAttributesSqlProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/sql/MqttAttributesSqlProtoIntegrationTest.java index 5f486916ae..c5eeb06546 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/sql/MqttAttributesSqlProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/attributes/sql/MqttAttributesSqlProtoIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.mqtt.telemetry.attributes.sql; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.mqtt.telemetry.attributes.AbstractMqttAttributesJsonIntegrationTest; import org.thingsboard.server.mqtt.telemetry.attributes.AbstractMqttAttributesProtoIntegrationTest; @DaoSqlTest diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java index f6294cd990..64ba73c374 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java @@ -27,7 +27,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.mqtt.AbstractMqttIntegrationTest; @@ -128,7 +127,34 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt } else { getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); } - Map>> values = doGetAsync(getTelemetryValuesUrl, Map.class); + start = System.currentTimeMillis(); + end = System.currentTimeMillis() + 5000; + Map>> values = null; + while (start <= end) { + values = doGetAsync(getTelemetryValuesUrl, Map.class); + boolean valid = values.size() == expectedKeys.size(); + if (valid) { + for (String key : expectedKeys) { + List> tsValues = values.get(key); + if (tsValues != null && tsValues.size() > 0) { + Object ts = tsValues.get(0).get("ts"); + if (ts == null) { + valid = false; + break; + } + } else { + valid = false; + break; + } + } + } + if (valid) { + break; + } + Thread.sleep(100); + start += 100; + } + assertNotNull(values); if (withTs) { assertTs(values, expectedKeys, 10000, 0); diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesJsonIntegrationTest.java index edab8405cd..181a35fa93 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesJsonIntegrationTest.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.device.profile.MqttTopics; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @Slf4j diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java index 2257350d31..4cb2f0271c 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java @@ -15,33 +15,36 @@ */ package org.thingsboard.server.mqtt.telemetry.timeseries; +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.TransportPayloadType; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.MqttTopics; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; @Slf4j public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends AbstractMqttTimeseriesIntegrationTest { private static final String POST_DATA_TELEMETRY_TOPIC = "proto/telemetry"; - @Before - public void beforeTest() throws Exception { - processBeforeTest("Test Post Telemetry device proto payload", "Test Post Telemetry gateway proto payload", TransportPayloadType.PROTOBUF, POST_DATA_TELEMETRY_TOPIC, null); - } - @After public void afterTest() throws Exception { processAfterTest(); @@ -49,20 +52,127 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac @Test public void testPushMqttTelemetry() throws Exception { + super.processBeforeTest("Test Post Telemetry device proto payload", "Test Post Telemetry gateway proto payload", TransportPayloadType.PROTOBUF, POST_DATA_TELEMETRY_TOPIC, null); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4", "key5"); - TransportProtos.TsKvListProto tsKvListProto = getTsKvListProto(expectedKeys, 0); - processTelemetryTest(POST_DATA_TELEMETRY_TOPIC, expectedKeys, tsKvListProto.toByteArray(), false); + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration); + MqttDeviceProfileTransportConfiguration mqttTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); + DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + + DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); + Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); + assertNotNull(nestedJsonObjectBuilderDescriptor); + DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build(); + + DynamicMessage.Builder jsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject"); + Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType(); + assertNotNull(jsonObjectBuilderDescriptor); + DynamicMessage jsonObject = jsonObjectBuilder + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3) + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject) + .build(); + + DynamicMessage.Builder postTelemetryBuilder = telemetrySchema.newMessageBuilder("PostTelemetry"); + Descriptors.Descriptor postTelemetryMsgDescriptor = postTelemetryBuilder.getDescriptorForType(); + assertNotNull(postTelemetryMsgDescriptor); + DynamicMessage postTelemetryMsg = postTelemetryBuilder + .setField(postTelemetryMsgDescriptor.findFieldByName("key1"), "value1") + .setField(postTelemetryMsgDescriptor.findFieldByName("key2"), true) + .setField(postTelemetryMsgDescriptor.findFieldByName("key3"), 3.0) + .setField(postTelemetryMsgDescriptor.findFieldByName("key4"), 4) + .setField(postTelemetryMsgDescriptor.findFieldByName("key5"), jsonObject) + .build(); + processTelemetryTest(POST_DATA_TELEMETRY_TOPIC, expectedKeys, postTelemetryMsg.toByteArray(), false); } @Test public void testPushMqttTelemetryWithTs() throws Exception { + String schemaStr = "syntax =\"proto3\";\n" + + "\n" + + "package test;\n" + + "\n" + + "message PostTelemetry {\n" + + " int64 ts = 1;\n" + + " Values values = 2;\n" + + " \n" + + " message Values {\n" + + " string key1 = 3;\n" + + " bool key2 = 4;\n" + + " double key3 = 5;\n" + + " int32 key4 = 6;\n" + + " JsonObject key5 = 7;\n" + + " }\n" + + " \n" + + " message JsonObject {\n" + + " int32 someNumber = 8;\n" + + " repeated int32 someArray = 9;\n" + + " NestedJsonObject someNestedObject = 10;\n" + + " message NestedJsonObject {\n" + + " string key = 11;\n" + + " }\n" + + " }\n" + + "}"; + super.processBeforeTest("Test Post Telemetry device proto payload", "Test Post Telemetry gateway proto payload", TransportPayloadType.PROTOBUF, POST_DATA_TELEMETRY_TOPIC, null, schemaStr, null, DeviceProfileProvisionType.DISABLED, null, null); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4", "key5"); - TransportProtos.TsKvListProto tsKvListProto = getTsKvListProto(expectedKeys, 10000); - processTelemetryTest(POST_DATA_TELEMETRY_TOPIC, expectedKeys, tsKvListProto.toByteArray(), true); + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration); + MqttDeviceProfileTransportConfiguration mqttTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(schemaStr); + DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + + DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); + Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); + assertNotNull(nestedJsonObjectBuilderDescriptor); + DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build(); + + DynamicMessage.Builder jsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject"); + Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType(); + assertNotNull(jsonObjectBuilderDescriptor); + DynamicMessage jsonObject = jsonObjectBuilder + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3) + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject) + .build(); + + + DynamicMessage.Builder valuesBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.Values"); + Descriptors.Descriptor valuesDescriptor = valuesBuilder.getDescriptorForType(); + assertNotNull(valuesDescriptor); + + DynamicMessage valuesMsg = valuesBuilder + .setField(valuesDescriptor.findFieldByName("key1"), "value1") + .setField(valuesDescriptor.findFieldByName("key2"), true) + .setField(valuesDescriptor.findFieldByName("key3"), 3.0) + .setField(valuesDescriptor.findFieldByName("key4"), 4) + .setField(valuesDescriptor.findFieldByName("key5"), jsonObject) + .build(); + + DynamicMessage.Builder postTelemetryBuilder = telemetrySchema.newMessageBuilder("PostTelemetry"); + Descriptors.Descriptor postTelemetryMsgDescriptor = postTelemetryBuilder.getDescriptorForType(); + assertNotNull(postTelemetryMsgDescriptor); + DynamicMessage postTelemetryMsg = postTelemetryBuilder + .setField(postTelemetryMsgDescriptor.findFieldByName("ts"), 10000L) + .setField(postTelemetryMsgDescriptor.findFieldByName("values"), valuesMsg) + .build(); + + processTelemetryTest(POST_DATA_TELEMETRY_TOPIC, expectedKeys, postTelemetryMsg.toByteArray(), true); } @Test public void testPushMqttTelemetryGateway() throws Exception { + super.processBeforeTest("Test Post Telemetry device proto payload", "Test Post Telemetry gateway proto payload", TransportPayloadType.PROTOBUF, null, null, null, null, DeviceProfileProvisionType.DISABLED, null, null); TransportApiProtos.GatewayTelemetryMsg.Builder gatewayTelemetryMsgProtoBuilder = TransportApiProtos.GatewayTelemetryMsg.newBuilder(); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4", "key5"); String deviceName1 = "Device A"; @@ -76,6 +186,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac @Test public void testGatewayConnect() throws Exception { + super.processBeforeTest("Test Post Telemetry device proto payload", "Test Post Telemetry gateway proto payload", TransportPayloadType.PROTOBUF, POST_DATA_TELEMETRY_TOPIC, null, null, null, DeviceProfileProvisionType.DISABLED, null, null); String deviceName = "Device A"; TransportApiProtos.ConnectMsg connectMsgProto = getConnectProto(deviceName); MqttAsyncClient client = getMqttAsyncClient(gatewayAccessToken); diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/sql/MqttTimeseriesSqlJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/sql/MqttTimeseriesSqlJsonIntegrationTest.java index f313cc5d29..0b494b6bea 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/sql/MqttTimeseriesSqlJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/sql/MqttTimeseriesSqlJsonIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.mqtt.telemetry.timeseries.sql; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.mqtt.telemetry.timeseries.AbstractMqttTimeseriesIntegrationTest; import org.thingsboard.server.mqtt.telemetry.timeseries.AbstractMqttTimeseriesJsonIntegrationTest; /** diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/sql/MqttTimeseriesSqlProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/sql/MqttTimeseriesSqlProtoIntegrationTest.java index 3c7d5e2e2a..2a80483879 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/sql/MqttTimeseriesSqlProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/timeseries/sql/MqttTimeseriesSqlProtoIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.mqtt.telemetry.timeseries.sql; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.mqtt.telemetry.timeseries.AbstractMqttTimeseriesJsonIntegrationTest; import org.thingsboard.server.mqtt.telemetry.timeseries.AbstractMqttTimeseriesProtoIntegrationTest; /** diff --git a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java deleted file mode 100644 index 60495b119d..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.script; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import com.google.common.collect.Sets; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.RuleNodeId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; - -import javax.script.ScriptException; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.*; - -public class RuleNodeJsScriptEngineTest { - - private ScriptEngine scriptEngine; - private TestNashornJsInvokeService jsSandboxService; - - private EntityId ruleNodeId = new RuleNodeId(Uuids.timeBased()); - - @Before - public void beforeTest() throws Exception { - jsSandboxService = new TestNashornJsInvokeService(false, 1, 100, 3); - } - - @After - public void afterTest() throws Exception { - jsSandboxService.stop(); - } - - @Test - public void msgCanBeUpdated() throws ScriptException { - String function = "metadata.temp = metadata.temp * 10; return {metadata: metadata};"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - - TbMsg actual = scriptEngine.executeUpdate(msg); - assertEquals("70", actual.getMetaData().getValue("temp")); - scriptEngine.destroy(); - } - - @Test - public void newAttributesCanBeAddedInMsg() throws ScriptException { - String function = "metadata.newAttr = metadata.humidity - msg.passed; return {metadata: metadata};"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - - TbMsg actual = scriptEngine.executeUpdate(msg); - assertEquals("94", actual.getMetaData().getValue("newAttr")); - scriptEngine.destroy(); - } - - @Test - public void payloadCanBeUpdated() throws ScriptException { - String function = "msg.passed = msg.passed * metadata.temp; msg.bigObj.newProp = 'Ukraine'; return {msg: msg};"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}"; - - TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); - - TbMsg actual = scriptEngine.executeUpdate(msg); - - String expectedJson = "{\"name\":\"Vit\",\"passed\":35,\"bigObj\":{\"prop\":42,\"newProp\":\"Ukraine\"}}"; - assertEquals(expectedJson, actual.getData()); - scriptEngine.destroy(); - } - - @Test - public void metadataAccessibleForFilter() throws ScriptException { - String function = "return metadata.humidity < 15;"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); - assertFalse(scriptEngine.executeFilter(msg)); - scriptEngine.destroy(); - } - - @Test - public void dataAccessibleForFilter() throws ScriptException { - String function = "return msg.passed < 15 && msg.name === 'Vit' && metadata.temp == 7 && msg.bigObj.prop == 42;"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson); - assertTrue(scriptEngine.executeFilter(msg)); - scriptEngine.destroy(); - } - - @Test - public void dataAccessibleForSwitch() throws ScriptException { - String jsCode = "function nextRelation(metadata, msg) {\n" + - " if(msg.passed == 5 && metadata.temp == 10)\n" + - " return 'one'\n" + - " else\n" + - " return 'two';\n" + - "};\n" + - "\n" + - "return nextRelation(metadata, msg);"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "10"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - Set actual = scriptEngine.executeSwitch(msg); - assertEquals(Sets.newHashSet("one"), actual); - scriptEngine.destroy(); - } - - @Test - public void multipleRelationsReturnedFromSwitch() throws ScriptException { - String jsCode = "function nextRelation(metadata, msg) {\n" + - " if(msg.passed == 5 && metadata.temp == 10)\n" + - " return ['three', 'one']\n" + - " else\n" + - " return 'two';\n" + - "};\n" + - "\n" + - "return nextRelation(metadata, msg);"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "10"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - Set actual = scriptEngine.executeSwitch(msg); - assertEquals(Sets.newHashSet("one", "three"), actual); - scriptEngine.destroy(); - } - - @Test - public void concurrentReleasedCorrectly() throws InterruptedException, ExecutionException { - String code = "metadata.temp = metadata.temp * 10; return {metadata: metadata};"; - - int repeat = 1000; - ExecutorService service = Executors.newFixedThreadPool(repeat); - Map scriptIds = new ConcurrentHashMap<>(); - CountDownLatch startLatch = new CountDownLatch(repeat); - CountDownLatch finishLatch = new CountDownLatch(repeat); - AtomicInteger failedCount = new AtomicInteger(0); - - for (int i = 0; i < repeat; i++) { - service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code)); - } - - finishLatch.await(); - assertTrue(scriptIds.size() == 1); - assertTrue(failedCount.get() == 0); - - CountDownLatch nextStart = new CountDownLatch(repeat); - CountDownLatch nextFinish = new CountDownLatch(repeat); - for (int i = 0; i < repeat; i++) { - service.submit(() -> runScript(nextStart, nextFinish, failedCount, scriptIds, code)); - } - - nextFinish.await(); - assertTrue(scriptIds.size() == 1); - assertTrue(failedCount.get() == 0); - service.shutdownNow(); - } - - @Test - public void concurrentFailedEvaluationShouldThrowException() throws InterruptedException { - String code = "metadata.temp = metadata.temp * 10; urn {metadata: metadata};"; - - int repeat = 10000; - ExecutorService service = Executors.newFixedThreadPool(repeat); - Map scriptIds = new ConcurrentHashMap<>(); - CountDownLatch startLatch = new CountDownLatch(repeat); - CountDownLatch finishLatch = new CountDownLatch(repeat); - AtomicInteger failedCount = new AtomicInteger(0); - for (int i = 0; i < repeat; i++) { - service.submit(() -> { - service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code)); - }); - } - - finishLatch.await(); - assertTrue(scriptIds.isEmpty()); - assertEquals(repeat, failedCount.get()); - service.shutdownNow(); - } - - private void runScript(CountDownLatch startLatch, CountDownLatch finishLatch, AtomicInteger failedCount, - Map scriptIds, String code) { - try { - for (int k = 0; k < 10; k++) { - startLatch.countDown(); - startLatch.await(); - UUID scriptId = jsSandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, code).get(); - scriptIds.put(scriptId, new Object()); - jsSandboxService.invokeFunction(scriptId, "{}", "{}", "TEXT").get(); - jsSandboxService.release(scriptId).get(); - } - } catch (Throwable th) { - failedCount.incrementAndGet(); - } finally { - finishLatch.countDown(); - } - } - -} diff --git a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java deleted file mode 100644 index 8a89de5dff..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.script; - -public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService { - - private boolean useJsSandbox; - private final int monitorThreadPoolSize; - private final long maxCpuTime; - private final int maxErrors; - - public TestNashornJsInvokeService(boolean useJsSandbox, int monitorThreadPoolSize, long maxCpuTime, int maxErrors) { - this.useJsSandbox = useJsSandbox; - this.monitorThreadPoolSize = monitorThreadPoolSize; - this.maxCpuTime = maxCpuTime; - this.maxErrors = maxErrors; - init(); - } - - @Override - protected boolean useJsSandbox() { - return useJsSandbox; - } - - @Override - protected int getMonitorThreadPoolSize() { - return monitorThreadPoolSize; - } - - @Override - protected long getMaxCpuTime() { - return maxCpuTime; - } - - @Override - protected int getMaxErrors() { - return maxErrors; - } - - @Override - protected long getMaxBlacklistDuration() { - return 100000; - } -} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceProfileService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceProfileService.java index e38bac68e5..b4ec680aaf 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceProfileService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceProfileService.java @@ -37,7 +37,7 @@ public interface DeviceProfileService { PageData findDeviceProfiles(TenantId tenantId, PageLink pageLink); - PageData findDeviceProfileInfos(TenantId tenantId, PageLink pageLink); + PageData findDeviceProfileInfos(TenantId tenantId, PageLink pageLink, String transportType); DeviceProfile findOrCreateDeviceProfile(TenantId tenantId, String profileName); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceProvisionService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceProvisionService.java new file mode 100644 index 0000000000..660f1a5f35 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceProvisionService.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.device; + +import org.thingsboard.server.dao.device.provision.ProvisionFailedException; +import org.thingsboard.server.dao.device.provision.ProvisionRequest; +import org.thingsboard.server.dao.device.provision.ProvisionResponse; + +public interface DeviceProvisionService { + + ProvisionResponse provisionDevice(ProvisionRequest provisionRequest) throws ProvisionFailedException; +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index dd55f2b884..bed9dc476a 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.device; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.id.CustomerId; @@ -28,6 +29,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.dao.device.provision.ProvisionRequest; import java.util.List; @@ -85,6 +87,8 @@ public interface DeviceService { Device assignDeviceToTenant(TenantId tenantId, Device device); + Device saveDevice(ProvisionRequest provisionRequest, DeviceProfile profile); + Device assignDeviceToEdge(TenantId tenantId, DeviceId deviceId, EdgeId edgeId); Device unassignDeviceFromEdge(TenantId tenantId, DeviceId deviceId, EdgeId edgeId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionFailedException.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionFailedException.java new file mode 100644 index 0000000000..04e8e00ce5 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionFailedException.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.device.provision; + +public class ProvisionFailedException extends RuntimeException { + public ProvisionFailedException(String errorMsg) { + super(errorMsg); + } +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionRequest.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionRequest.java new file mode 100644 index 0000000000..f45b541074 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionRequest.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.device.provision; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.device.credentials.ProvisionDeviceCredentialsData; +import org.thingsboard.server.common.data.device.profile.ProvisionDeviceProfileCredentials; +import org.thingsboard.server.common.data.security.DeviceCredentialsType; + +@Data +@AllArgsConstructor +public class ProvisionRequest { + private String deviceName; + private DeviceCredentialsType credentialsType; + private ProvisionDeviceCredentialsData credentialsData; + private ProvisionDeviceProfileCredentials credentials; +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionResponse.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionResponse.java new file mode 100644 index 0000000000..d38b53b134 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionResponse.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.device.provision; + +import lombok.Data; +import org.thingsboard.server.common.data.security.DeviceCredentials; + +@Data +public class ProvisionResponse { + private final DeviceCredentials deviceCredentials; + private final ProvisionResponseStatus responseStatus; +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionResponseStatus.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionResponseStatus.java new file mode 100644 index 0000000000..d4fbe65a97 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/provision/ProvisionResponseStatus.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.device.provision; + +public enum ProvisionResponseStatus { + UNKNOWN, + SUCCESS, + NOT_FOUND, + FAILURE +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ConfigTemplateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ConfigTemplateService.java new file mode 100644 index 0000000000..54a47d9fdb --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ConfigTemplateService.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import org.thingsboard.server.common.data.id.OAuth2ClientRegistrationTemplateId; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationTemplate; + +import java.util.List; +import java.util.Optional; + +public interface OAuth2ConfigTemplateService { + OAuth2ClientRegistrationTemplate saveClientRegistrationTemplate(OAuth2ClientRegistrationTemplate clientRegistrationTemplate); + + Optional findClientRegistrationTemplateByProviderId(String providerId); + + OAuth2ClientRegistrationTemplate findClientRegistrationTemplateById(OAuth2ClientRegistrationTemplateId templateId); + + List findAllClientRegistrationTemplates(); + + void deleteClientRegistrationTemplateById(OAuth2ClientRegistrationTemplateId templateId); +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java index d72b6ef98c..073bab823b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java @@ -16,10 +16,20 @@ package org.thingsboard.server.dao.oauth2; import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientsParams; import java.util.List; +import java.util.UUID; public interface OAuth2Service { + List getOAuth2Clients(String domainScheme, String domainName); - List getOAuth2Clients(); + void saveOAuth2Params(OAuth2ClientsParams oauth2Params); + + OAuth2ClientsParams findOAuth2Params(); + + OAuth2ClientRegistrationInfo findClientRegistrationInfo(UUID id); + + List findAllClientRegistrationInfos(); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java index 07138a1a11..924cad0884 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java @@ -30,4 +30,7 @@ public interface RuleNodeStateService { RuleNodeState save(TenantId tenantId, RuleNodeState ruleNodeState); + void removeByRuleNodeId(TenantId tenantId, RuleNodeId selfId); + + void removeByRuleNodeIdAndEntityId(TenantId tenantId, RuleNodeId selfId, EntityId entityId); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java index ee263c2564..a0c8d46d72 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java @@ -24,7 +24,7 @@ public interface AdminSettingsService { AdminSettings findAdminSettingsById(TenantId tenantId, AdminSettingsId adminSettingsId); AdminSettings findAdminSettingsByKey(TenantId tenantId, String key); - + AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings); - + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TbTenantProfileCache.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TbTenantProfileCache.java new file mode 100644 index 0000000000..ea3d65eb49 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TbTenantProfileCache.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; + +import java.util.function.Consumer; + +public interface TbTenantProfileCache { + + TenantProfile get(TenantId tenantId); + + TenantProfile get(TenantProfileId tenantProfileId); + + void put(TenantProfile profile); + + void evict(TenantProfileId id); + + void evict(TenantId id); + + void addListener(TenantId tenantId, EntityId listenerId, Consumer profileListener); + + void removeListener(TenantId tenantId, EntityId listenerId); + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 2a5a701f9e..0e11aedaf5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -36,9 +36,9 @@ public interface TimeseriesService { ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture> save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java new file mode 100644 index 0000000000..5388cab283 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.usagerecord; + +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface ApiUsageStateService { + + ApiUsageState createDefaultApiUsageState(TenantId id); + + ApiUsageState update(ApiUsageState apiUsageState); + + ApiUsageState findTenantApiUsageState(TenantId tenantId); + + void deleteApiUsageStateByTenantId(TenantId tenantId); + + ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id); +} diff --git a/common/data/pom.xml b/common/data/pom.xml index c57cbacfa9..6fd21cd9f3 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -71,6 +71,14 @@ java-driver-core test + + com.squareup.wire + wire-schema + + + org.thingsboard + protobuf-dynamic + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiFeature.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiFeature.java new file mode 100644 index 0000000000..10ae01a699 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiFeature.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.Getter; + +public enum ApiFeature { + TRANSPORT("transportApiState", "Device API"), + DB("dbApiState", "Telemetry persistence"), + RE("ruleEngineApiState", "Rule Engine execution"), + JS("jsExecutionApiState", "JavaScript functions execution"), + EMAIL("emailApiState", "Email messages"), + SMS("smsApiState", "SMS messages"); + + @Getter + private final String apiStateKey; + @Getter + private final String label; + + ApiFeature(String apiStateKey, String label) { + this.apiStateKey = apiStateKey; + this.label = label; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java new file mode 100644 index 0000000000..70b2902256 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.Getter; + +public enum ApiUsageRecordKey { + + TRANSPORT_MSG_COUNT(ApiFeature.TRANSPORT, "transportMsgCount", "transportMsgLimit"), + TRANSPORT_DP_COUNT(ApiFeature.TRANSPORT, "transportDataPointsCount", "transportDataPointsLimit"), + STORAGE_DP_COUNT(ApiFeature.DB, "storageDataPointsCount", "storageDataPointsLimit"), + RE_EXEC_COUNT(ApiFeature.RE, "ruleEngineExecutionCount", "ruleEngineExecutionLimit"), + JS_EXEC_COUNT(ApiFeature.JS, "jsExecutionCount", "jsExecutionLimit"), + EMAIL_EXEC_COUNT(ApiFeature.EMAIL, "emailCount", "emailLimit"), + SMS_EXEC_COUNT(ApiFeature.SMS, "smsCount", "smsLimit"); + private static final ApiUsageRecordKey[] JS_RECORD_KEYS = {JS_EXEC_COUNT}; + private static final ApiUsageRecordKey[] RE_RECORD_KEYS = {RE_EXEC_COUNT}; + private static final ApiUsageRecordKey[] DB_RECORD_KEYS = {STORAGE_DP_COUNT}; + private static final ApiUsageRecordKey[] TRANSPORT_RECORD_KEYS = {TRANSPORT_MSG_COUNT, TRANSPORT_DP_COUNT}; + private static final ApiUsageRecordKey[] EMAIL_RECORD_KEYS = {EMAIL_EXEC_COUNT}; + private static final ApiUsageRecordKey[] SMS_RECORD_KEYS = {SMS_EXEC_COUNT}; + + @Getter + private final ApiFeature apiFeature; + @Getter + private final String apiCountKey; + @Getter + private final String apiLimitKey; + + ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey) { + this.apiFeature = apiFeature; + this.apiCountKey = apiCountKey; + this.apiLimitKey = apiLimitKey; + } + + public static ApiUsageRecordKey[] getKeys(ApiFeature feature) { + switch (feature) { + case TRANSPORT: + return TRANSPORT_RECORD_KEYS; + case DB: + return DB_RECORD_KEYS; + case RE: + return RE_RECORD_KEYS; + case JS: + return JS_RECORD_KEYS; + case EMAIL: + return EMAIL_RECORD_KEYS; + case SMS: + return SMS_RECORD_KEYS; + default: + return new ApiUsageRecordKey[]{}; + } + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java new file mode 100644 index 0000000000..8f13fa8164 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java @@ -0,0 +1,100 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.ApiUsageStateId; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class ApiUsageState extends BaseData implements HasTenantId { + + private static final long serialVersionUID = 8250339805336035966L; + + @Getter + @Setter + private TenantId tenantId; + @Getter + @Setter + private EntityId entityId; + @Getter + @Setter + private ApiUsageStateValue transportState; + @Getter + @Setter + private ApiUsageStateValue dbStorageState; + @Getter + @Setter + private ApiUsageStateValue reExecState; + @Getter + @Setter + private ApiUsageStateValue jsExecState; + @Getter + @Setter + private ApiUsageStateValue emailExecState; + @Getter + @Setter + private ApiUsageStateValue smsExecState; + + public ApiUsageState() { + super(); + } + + public ApiUsageState(ApiUsageStateId id) { + super(id); + } + + public ApiUsageState(ApiUsageState ur) { + super(ur); + this.tenantId = ur.getTenantId(); + this.entityId = ur.getEntityId(); + this.transportState = ur.getTransportState(); + this.dbStorageState = ur.getDbStorageState(); + this.reExecState = ur.getReExecState(); + this.jsExecState = ur.getJsExecState(); + this.emailExecState = ur.getEmailExecState(); + this.smsExecState = ur.getSmsExecState(); + } + + public boolean isTransportEnabled() { + return !ApiUsageStateValue.DISABLED.equals(transportState); + } + + public boolean isReExecEnabled() { + return !ApiUsageStateValue.DISABLED.equals(reExecState); + } + + public boolean isDbStorageEnabled() { + return !ApiUsageStateValue.DISABLED.equals(dbStorageState); + } + + public boolean isJsExecEnabled() { + return !ApiUsageStateValue.DISABLED.equals(jsExecState); + } + + public boolean isEmailSendEnabled(){ + return !ApiUsageStateValue.DISABLED.equals(emailExecState); + } + + public boolean isSmsSendEnabled(){ + return !ApiUsageStateValue.DISABLED.equals(smsExecState); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageStateMailMessage.java similarity index 55% rename from common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageStateMailMessage.java index 54319276f1..cab3789fdc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageStateMailMessage.java @@ -15,28 +15,11 @@ */ package org.thingsboard.server.common.data; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; -import java.util.HashMap; -import java.util.Map; - @Data -public class TenantProfileData { - - @JsonIgnore - private Map properties = new HashMap<>(); - - @JsonAnyGetter - public Map properties() { - return this.properties; - } - - @JsonAnySetter - public void put(String name, Object value) { - this.properties.put(name, value); - } - +public class ApiUsageStateMailMessage { + private final ApiUsageRecordKey key; + private final long threshold; + private final long value; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageStateValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageStateValue.java new file mode 100644 index 0000000000..ae14ff80bf --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageStateValue.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +public enum ApiUsageStateValue { + + ENABLED, WARNING, DISABLED; + + + public static ApiUsageStateValue toMoreRestricted(ApiUsageStateValue a, ApiUsageStateValue b) { + return a.ordinal() > b.ordinal() ? a : b; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 0d225a4b8f..41beb9cf57 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -63,6 +63,8 @@ public class DataConstants { public static final String ALARM_CLEAR = "ALARM_CLEAR"; public static final String ENTITY_ASSIGNED_FROM_TENANT = "ENTITY_ASSIGNED_FROM_TENANT"; public static final String ENTITY_ASSIGNED_TO_TENANT = "ENTITY_ASSIGNED_TO_TENANT"; + public static final String PROVISION_SUCCESS = "PROVISION_SUCCESS"; + public static final String PROVISION_FAILURE = "PROVISION_FAILURE"; public static final String ENTITY_ASSIGNED_TO_EDGE = "ENTITY_ASSIGNED_TO_EDGE"; public static final String ENTITY_UNASSIGNED_FROM_EDGE = "ENTITY_UNASSIGNED_FROM_EDGE"; @@ -72,6 +74,20 @@ public class DataConstants { public static final String SECRET_KEY_FIELD_NAME = "secretKey"; public static final String DURATION_MS_FIELD_NAME = "durationMs"; + public static final String PROVISION = "provision"; + public static final String PROVISION_KEY = "provisionDeviceKey"; + public static final String PROVISION_SECRET = "provisionDeviceSecret"; + + public static final String DEVICE_NAME = "deviceName"; + public static final String DEVICE_TYPE = "deviceType"; + public static final String CERT_PUB_KEY = "x509CertPubKey"; + public static final String CREDENTIALS_TYPE = "credentialsType"; + public static final String TOKEN = "token"; + public static final String HASH = "hash"; + public static final String CLIENT_ID = "clientId"; + public static final String USERNAME = "username"; + public static final String PASSWORD = "password"; + public static final String EDGE_MSG_SOURCE = "edge"; public static final String MSG_SOURCE_KEY = "source"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java index 097d64b198..b9ecb862eb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java @@ -41,10 +41,13 @@ public class DeviceProfile extends SearchTextBased implements H private boolean isDefault; private DeviceProfileType type; private DeviceTransportType transportType; + private DeviceProfileProvisionType provisionType; private RuleChainId defaultRuleChainId; + private String defaultQueueName; private transient DeviceProfileData profileData; @JsonIgnore private byte[] profileDataBytes; + private String provisionDeviceKey; public DeviceProfile() { super(); @@ -61,7 +64,9 @@ public class DeviceProfile extends SearchTextBased implements H this.description = deviceProfile.getDescription(); this.isDefault = deviceProfile.isDefault(); this.defaultRuleChainId = deviceProfile.getDefaultRuleChainId(); + this.defaultQueueName = deviceProfile.getDefaultQueueName(); this.setProfileData(deviceProfile.getProfileData()); + this.provisionDeviceKey = deviceProfile.getProvisionDeviceKey(); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfileProvisionType.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfileProvisionType.java new file mode 100644 index 0000000000..f33c705d52 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfileProvisionType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +public enum DeviceProfileProvisionType { + DISABLED, + ALLOW_CREATE_NEW_DEVICES, + CHECK_PRE_PROVISIONED_DEVICES +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index e02386db1f..9ef65ca366 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -19,5 +19,5 @@ package org.thingsboard.server.common.data; * @author Andrew Shvayka */ public enum EntityType { - TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, EDGE + TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, API_USAGE_STATE, EDGE; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java index 32f620f869..acb9c239b8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java @@ -21,6 +21,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -78,15 +80,21 @@ public class TenantProfile extends SearchTextBased implements H profileData = mapper.readValue(new ByteArrayInputStream(profileDataBytes), TenantProfileData.class); } catch (IOException e) { log.warn("Can't deserialize tenant profile data: ", e); - return null; + return createDefaultTenantProfileData(); } return profileData; } else { - return null; + return createDefaultTenantProfileData(); } } } + public TenantProfileData createDefaultTenantProfileData() { + TenantProfileData tpd = new TenantProfileData(); + tpd.setConfiguration(new DefaultTenantProfileConfiguration()); + return tpd; + } + public void setProfileData(TenantProfileData data) { this.profileData = data; try { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java new file mode 100644 index 0000000000..2b0ed502ac --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +public enum TenantProfileType { + DEFAULT +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java b/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java index 6b6706bc84..0fe921968f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java @@ -44,6 +44,8 @@ public enum ActionType { LOCKOUT(false), ASSIGNED_FROM_TENANT(false), ASSIGNED_TO_TENANT(false), + PROVISION_SUCCESS(false), + PROVISION_FAILURE(false), ASSIGNED_TO_EDGE(false), // log edge name UNASSIGNED_FROM_EDGE(false), // log edge name CREDENTIALS_REQUEST(false), // request credentials from edge diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/ProvisionDeviceCredentialsData.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/ProvisionDeviceCredentialsData.java new file mode 100644 index 0000000000..fe6cc2621a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/ProvisionDeviceCredentialsData.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.credentials; + +import lombok.Data; + +@Data +public class ProvisionDeviceCredentialsData { + private final String token; + private final String clientId; + private final String username; + private final String password; + private final String x509CertHash; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AllowCreateNewDevicesDeviceProfileProvisionConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AllowCreateNewDevicesDeviceProfileProvisionConfiguration.java new file mode 100644 index 0000000000..8414caa400 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AllowCreateNewDevicesDeviceProfileProvisionConfiguration.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import lombok.Data; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; + +@Data +public class AllowCreateNewDevicesDeviceProfileProvisionConfiguration implements DeviceProfileProvisionConfiguration { + + private final String provisionDeviceSecret; + + @Override + public DeviceProfileProvisionType getType() { + return DeviceProfileProvisionType.ALLOW_CREATE_NEW_DEVICES; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CheckPreProvisionedDevicesDeviceProfileProvisionConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CheckPreProvisionedDevicesDeviceProfileProvisionConfiguration.java new file mode 100644 index 0000000000..40af29ea51 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CheckPreProvisionedDevicesDeviceProfileProvisionConfiguration.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import lombok.Data; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; + +@Data +public class CheckPreProvisionedDevicesDeviceProfileProvisionConfiguration implements DeviceProfileProvisionConfiguration { + + private final String provisionDeviceSecret; + + @Override + public DeviceProfileProvisionType getType() { + return DeviceProfileProvisionType.CHECK_PRE_PROVISIONED_DEVICES; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java index b6437ae7bb..3b6acce6f3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java @@ -19,7 +19,7 @@ import lombok.Data; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import java.util.List; -import java.util.Map; +import java.util.TreeMap; @Data public class DeviceProfileAlarm { @@ -27,7 +27,7 @@ public class DeviceProfileAlarm { private String id; private String alarmType; - private Map createRules; + private TreeMap createRules; private AlarmRule clearRule; // Hidden in advanced settings diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileData.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileData.java index 275e6269d6..7cb3304934 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileData.java @@ -24,6 +24,7 @@ public class DeviceProfileData { private DeviceProfileConfiguration configuration; private DeviceProfileTransportConfiguration transportConfiguration; + private DeviceProfileProvisionConfiguration provisionConfiguration; private List alarms; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileProvisionConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileProvisionConfiguration.java new file mode 100644 index 0000000000..f892aabf76 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileProvisionConfiguration.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; + + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = DisabledDeviceProfileProvisionConfiguration.class, name = "DISABLED"), + @JsonSubTypes.Type(value = AllowCreateNewDevicesDeviceProfileProvisionConfiguration.class, name = "ALLOW_CREATE_NEW_DEVICES"), + @JsonSubTypes.Type(value = CheckPreProvisionedDevicesDeviceProfileProvisionConfiguration.class, name = "CHECK_PRE_PROVISIONED_DEVICES")}) +public interface DeviceProfileProvisionConfiguration { + + String getProvisionDeviceSecret(); + + @JsonIgnore + DeviceProfileProvisionType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java index 34854958d1..fb337b24f4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java @@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.DeviceTransportType; property = "type") @JsonSubTypes({ @JsonSubTypes.Type(value = DefaultDeviceProfileTransportConfiguration.class, name = "DEFAULT"), - @JsonSubTypes.Type(value = MqttDeviceProfileTransportConfiguration.class, name = "MQTT"), + @JsonSubTypes.Type(value = MqttDeviceProfileTransportConfiguration.class, name = "MQTT"), @JsonSubTypes.Type(value = Lwm2mDeviceProfileTransportConfiguration.class, name = "LWM2M")}) public interface DeviceProfileTransportConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DisabledDeviceProfileProvisionConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DisabledDeviceProfileProvisionConfiguration.java new file mode 100644 index 0000000000..01876c810d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DisabledDeviceProfileProvisionConfiguration.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import lombok.Data; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; + +@Data +public class DisabledDeviceProfileProvisionConfiguration implements DeviceProfileProvisionConfiguration { + + private final String provisionDeviceSecret; + + @Override + public DeviceProfileProvisionType getType() { + return DeviceProfileProvisionType.DISABLED; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/JsonTransportPayloadConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/JsonTransportPayloadConfiguration.java new file mode 100644 index 0000000000..f4fd93459b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/JsonTransportPayloadConfiguration.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import lombok.Data; +import org.thingsboard.server.common.data.TransportPayloadType; + +@Data +public class JsonTransportPayloadConfiguration implements TransportPayloadTypeConfiguration { + + @Override + public TransportPayloadType getTransportPayloadType() { + return TransportPayloadType.JSON; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttDeviceProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttDeviceProfileTransportConfiguration.java index d88ac24cbb..c49b7bd356 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttDeviceProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttDeviceProfileTransportConfiguration.java @@ -16,20 +16,27 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; -import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.DeviceTransportType; @Data public class MqttDeviceProfileTransportConfiguration implements DeviceProfileTransportConfiguration { - private TransportPayloadType transportPayloadType = TransportPayloadType.JSON; - private String deviceTelemetryTopic = MqttTopics.DEVICE_TELEMETRY_TOPIC; private String deviceAttributesTopic = MqttTopics.DEVICE_ATTRIBUTES_TOPIC; + private TransportPayloadTypeConfiguration transportPayloadTypeConfiguration; @Override public DeviceTransportType getType() { return DeviceTransportType.MQTT; } + public TransportPayloadTypeConfiguration getTransportPayloadTypeConfiguration() { + if (transportPayloadTypeConfiguration != null) { + return transportPayloadTypeConfiguration; + } else { + return new JsonTransportPayloadConfiguration(); + } + } + + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java index ba8ac7f3be..b226b87999 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java @@ -20,6 +20,8 @@ package org.thingsboard.server.common.data.device.profile; */ public class MqttTopics { + private static final String REQUEST = "/request"; + private static final String RESPONSE = "/response"; private static final String RPC = "/rpc"; private static final String CONNECT = "/connect"; private static final String DISCONNECT = "/disconnect"; @@ -27,11 +29,13 @@ public class MqttTopics { private static final String ATTRIBUTES = "/attributes"; private static final String CLAIM = "/claim"; private static final String SUB_TOPIC = "+"; - private static final String ATTRIBUTES_RESPONSE = "/attributes/response"; - private static final String ATTRIBUTES_REQUEST = "/attributes/request"; + private static final String PROVISION = "/provision"; - private static final String DEVICE_RPC_RESPONSE = "/rpc/response/"; - private static final String DEVICE_RPC_REQUEST = "/rpc/request/"; + private static final String ATTRIBUTES_RESPONSE = ATTRIBUTES + RESPONSE; + private static final String ATTRIBUTES_REQUEST = ATTRIBUTES + REQUEST; + + private static final String DEVICE_RPC_RESPONSE = RPC + RESPONSE + "/"; + private static final String DEVICE_RPC_REQUEST = RPC + REQUEST + "/"; private static final String DEVICE_ATTRIBUTES_RESPONSE = ATTRIBUTES_RESPONSE + "/"; private static final String DEVICE_ATTRIBUTES_REQUEST = ATTRIBUTES_REQUEST + "/"; @@ -50,6 +54,8 @@ public class MqttTopics { public static final String DEVICE_TELEMETRY_TOPIC = BASE_DEVICE_API_TOPIC + TELEMETRY; public static final String DEVICE_CLAIM_TOPIC = BASE_DEVICE_API_TOPIC + CLAIM; public static final String DEVICE_ATTRIBUTES_TOPIC = BASE_DEVICE_API_TOPIC + ATTRIBUTES; + public static final String DEVICE_PROVISION_REQUEST_TOPIC = PROVISION + REQUEST; + public static final String DEVICE_PROVISION_RESPONSE_TOPIC = PROVISION + RESPONSE; // V1_JSON gateway topics diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java new file mode 100644 index 0000000000..08c846d905 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java @@ -0,0 +1,200 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.github.os72.protobuf.dynamic.EnumDefinition; +import com.github.os72.protobuf.dynamic.MessageDefinition; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.squareup.wire.schema.Location; +import com.squareup.wire.schema.internal.parser.EnumConstantElement; +import com.squareup.wire.schema.internal.parser.EnumElement; +import com.squareup.wire.schema.internal.parser.FieldElement; +import com.squareup.wire.schema.internal.parser.MessageElement; +import com.squareup.wire.schema.internal.parser.OneOfElement; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; +import com.squareup.wire.schema.internal.parser.ProtoParser; +import com.squareup.wire.schema.internal.parser.TypeElement; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.TransportPayloadType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +@Data +public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeConfiguration { + + public static final Location LOCATION = new Location("", "", -1, -1); + public static final String ATTRIBUTES_PROTO_SCHEMA = "attributes proto schema"; + public static final String TELEMETRY_PROTO_SCHEMA = "telemetry proto schema"; + + private String deviceTelemetryProtoSchema; + private String deviceAttributesProtoSchema; + + @Override + public TransportPayloadType getTransportPayloadType() { + return TransportPayloadType.PROTOBUF; + } + + public Descriptors.Descriptor getTelemetryDynamicMessageDescriptor(String deviceTelemetryProtoSchema) { + return getDescriptor(deviceTelemetryProtoSchema, TELEMETRY_PROTO_SCHEMA); + } + + public Descriptors.Descriptor getAttributesDynamicMessageDescriptor(String deviceAttributesProtoSchema) { + return getDescriptor(deviceAttributesProtoSchema, ATTRIBUTES_PROTO_SCHEMA); + } + + private Descriptors.Descriptor getDescriptor(String protoSchema, String schemaName) { + try { + ProtoFileElement protoFileElement = getTransportProtoSchema(protoSchema); + DynamicSchema dynamicSchema = getDynamicSchema(protoFileElement, schemaName); + String lastMsgName = getMessageTypes(protoFileElement.getTypes()).stream() + .map(MessageElement::getName).reduce((previous, last) -> last).get(); + DynamicMessage.Builder builder = dynamicSchema.newMessageBuilder(lastMsgName); + return builder.getDescriptorForType(); + } catch (Exception e) { + log.warn("Failed to get Message Descriptor due to {}", e.getMessage()); + return null; + } + } + + public DynamicSchema getDynamicSchema(ProtoFileElement protoFileElement, String schemaName) { + DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); + schemaBuilder.setName(schemaName); + schemaBuilder.setPackage(!isEmptyStr(protoFileElement.getPackageName()) ? + protoFileElement.getPackageName() : schemaName.toLowerCase()); + List types = protoFileElement.getTypes(); + List messageTypes = getMessageTypes(types); + + if (!messageTypes.isEmpty()) { + List enumTypes = getEnumElements(types); + if (!enumTypes.isEmpty()) { + enumTypes.forEach(enumElement -> { + EnumDefinition enumDefinition = getEnumDefinition(enumElement); + schemaBuilder.addEnumDefinition(enumDefinition); + }); + } + List messageDefinitions = getMessageDefinitions(messageTypes); + messageDefinitions.forEach(schemaBuilder::addMessageDefinition); + try { + return schemaBuilder.build(); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException("Failed to create dynamic schema due to: " + e.getMessage()); + } + } else { + throw new RuntimeException("Failed to get Dynamic Schema! Message types is empty for schema:" + schemaName); + } + } + + public ProtoFileElement getTransportProtoSchema(String protoSchema) { + return new ProtoParser(LOCATION, protoSchema.toCharArray()).readProtoFile(); + } + + private List getMessageTypes(List types) { + return types.stream() + .filter(typeElement -> typeElement instanceof MessageElement) + .map(typeElement -> (MessageElement) typeElement) + .collect(Collectors.toList()); + } + + private List getEnumElements(List types) { + return types.stream() + .filter(typeElement -> typeElement instanceof EnumElement) + .map(typeElement -> (EnumElement) typeElement) + .collect(Collectors.toList()); + } + + private List getMessageDefinitions(List messageElementsList) { + if (!messageElementsList.isEmpty()) { + List messageDefinitions = new ArrayList<>(); + messageElementsList.forEach(messageElement -> { + MessageDefinition.Builder messageDefinitionBuilder = MessageDefinition.newBuilder(messageElement.getName()); + + List nestedTypes = messageElement.getNestedTypes(); + if (!nestedTypes.isEmpty()) { + List nestedEnumTypes = getEnumElements(nestedTypes); + if (!nestedEnumTypes.isEmpty()) { + nestedEnumTypes.forEach(enumElement -> { + EnumDefinition nestedEnumDefinition = getEnumDefinition(enumElement); + messageDefinitionBuilder.addEnumDefinition(nestedEnumDefinition); + }); + } + List nestedMessageTypes = getMessageTypes(nestedTypes); + List nestedMessageDefinitions = getMessageDefinitions(nestedMessageTypes); + nestedMessageDefinitions.forEach(messageDefinitionBuilder::addMessageDefinition); + } + List messageElementFields = messageElement.getFields(); + List oneOfs = messageElement.getOneOfs(); + if (!oneOfs.isEmpty()) { + for (OneOfElement oneOfelement : oneOfs) { + MessageDefinition.OneofBuilder oneofBuilder = messageDefinitionBuilder.addOneof(oneOfelement.getName()); + addMessageFieldsToTheOneOfDefinition(oneOfelement.getFields(), oneofBuilder); + } + } + if (!messageElementFields.isEmpty()) { + addMessageFieldsToTheMessageDefinition(messageElementFields, messageDefinitionBuilder); + } + messageDefinitions.add(messageDefinitionBuilder.build()); + }); + return messageDefinitions; + } else { + return Collections.emptyList(); + } + } + + private EnumDefinition getEnumDefinition(EnumElement enumElement) { + List enumElementTypeConstants = enumElement.getConstants(); + EnumDefinition.Builder enumDefinitionBuilder = EnumDefinition.newBuilder(enumElement.getName()); + if (!enumElementTypeConstants.isEmpty()) { + enumElementTypeConstants.forEach(constantElement -> enumDefinitionBuilder.addValue(constantElement.getName(), constantElement.getTag())); + } + return enumDefinitionBuilder.build(); + } + + + private void addMessageFieldsToTheMessageDefinition(List messageElementFields, MessageDefinition.Builder messageDefinitionBuilder) { + messageElementFields.forEach(fieldElement -> { + String labelStr = null; + if (fieldElement.getLabel() != null) { + labelStr = fieldElement.getLabel().name().toLowerCase(); + } + messageDefinitionBuilder.addField( + labelStr, + fieldElement.getType(), + fieldElement.getName(), + fieldElement.getTag()); + }); + } + + private void addMessageFieldsToTheOneOfDefinition(List oneOfsElementFields, MessageDefinition.OneofBuilder oneofBuilder) { + oneOfsElementFields.forEach(fieldElement -> oneofBuilder.addField( + fieldElement.getType(), + fieldElement.getName(), + fieldElement.getTag())); + oneofBuilder.msgDefBuilder(); + } + + private boolean isEmptyStr(String str) { + return str == null || "".equals(str); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProvisionDeviceProfileCredentials.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProvisionDeviceProfileCredentials.java new file mode 100644 index 0000000000..659da71488 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProvisionDeviceProfileCredentials.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import lombok.Data; + +@Data +public class ProvisionDeviceProfileCredentials { + private final String provisionDeviceKey; + private final String provisionDeviceSecret; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/TransportPayloadTypeConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/TransportPayloadTypeConfiguration.java new file mode 100644 index 0000000000..5a6bbd5b25 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/TransportPayloadTypeConfiguration.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.TransportPayloadType; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "transportPayloadType") +@JsonSubTypes({ + @JsonSubTypes.Type(value = JsonTransportPayloadConfiguration.class, name = "JSON"), + @JsonSubTypes.Type(value = ProtoTransportPayloadConfiguration.class, name = "PROTOBUF")}) +public interface TransportPayloadTypeConfiguration { + + @JsonIgnore + TransportPayloadType getTransportPayloadType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java new file mode 100644 index 0000000000..87983f519a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +public class ApiUsageStateId extends UUIDBased implements EntityId { + + @JsonCreator + public ApiUsageStateId(@JsonProperty("id") UUID id) { + super(id); + } + + public static ApiUsageStateId fromString(String userId) { + return new ApiUsageStateId(UUID.fromString(userId)); + } + + @JsonIgnore + @Override + public EntityType getEntityType() { + return EntityType.API_USAGE_STATE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index b3713947fd..74f3970a0b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -67,6 +67,8 @@ public class EntityIdFactory { return new DeviceProfileId(uuid); case TENANT_PROFILE: return new TenantProfileId(uuid); + case API_USAGE_STATE: + return new ApiUsageStateId(uuid); case EDGE: return new EdgeId(uuid); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2IntegrationId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2ClientRegistrationId.java similarity index 72% rename from common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2IntegrationId.java rename to common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2ClientRegistrationId.java index 30fd55d204..e87d9c7b2e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2IntegrationId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2ClientRegistrationId.java @@ -20,16 +20,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.UUID; -public class OAuth2IntegrationId extends UUIDBased { - - private static final long serialVersionUID = 1L; +public class OAuth2ClientRegistrationId extends UUIDBased { @JsonCreator - public OAuth2IntegrationId(@JsonProperty("id") UUID id) { + public OAuth2ClientRegistrationId(@JsonProperty("id") UUID id) { super(id); } - public static OAuth2IntegrationId fromString(String oauth2IntegrationId) { - return new OAuth2IntegrationId(UUID.fromString(oauth2IntegrationId)); + public static OAuth2ClientRegistrationId fromString(String clientRegistrationId) { + return new OAuth2ClientRegistrationId(UUID.fromString(clientRegistrationId)); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2ClientRegistrationInfoId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2ClientRegistrationInfoId.java new file mode 100644 index 0000000000..87a2109909 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2ClientRegistrationInfoId.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.UUID; + +public class OAuth2ClientRegistrationInfoId extends UUIDBased { + + @JsonCreator + public OAuth2ClientRegistrationInfoId(@JsonProperty("id") UUID id) { + super(id); + } + + public static OAuth2ClientRegistrationInfoId fromString(String clientRegistrationInfoId) { + return new OAuth2ClientRegistrationInfoId(UUID.fromString(clientRegistrationInfoId)); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2ClientRegistrationTemplateId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2ClientRegistrationTemplateId.java new file mode 100644 index 0000000000..cc6ee3a795 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2ClientRegistrationTemplateId.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.UUID; + +public class OAuth2ClientRegistrationTemplateId extends UUIDBased { + + @JsonCreator + public OAuth2ClientRegistrationTemplateId(@JsonProperty("id") UUID id) { + super(id); + } + + public static OAuth2ClientRegistrationTemplateId fromString(String clientRegistrationTemplateId) { + return new OAuth2ClientRegistrationTemplateId(UUID.fromString(clientRegistrationTemplateId)); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java index c2d6688004..bcf49899dc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java @@ -19,7 +19,7 @@ import java.util.Objects; import java.util.Optional; public class BasicTsKvEntry implements TsKvEntry { - + private static final int MAX_CHARS_PER_DATA_POINT = 512; private final long ts; private final KvEntry kv; @@ -99,4 +99,21 @@ public class BasicTsKvEntry implements TsKvEntry { public String getValueAsString() { return kv.getValueAsString(); } + + @Override + public int getDataPoints() { + int length; + switch (getDataType()) { + case STRING: + length = getStrValue().get().length(); + break; + case JSON: + length = getJsonValue().get().length(); + break; + default: + return 1; + } + return Math.max(1, (length + MAX_CHARS_PER_DATA_POINT - 1) / MAX_CHARS_PER_DATA_POINT); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java index 315d3940df..e2f8eaf1e5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.kv; +import com.fasterxml.jackson.annotation.JsonIgnore; + /** * Represents time series KV data entry * @@ -25,4 +27,7 @@ public interface TsKvEntry extends KvEntry { long getTs(); + @JsonIgnore + int getDataPoints(); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Client.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/ClientRegistrationDto.java similarity index 67% rename from dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Client.java rename to common/data/src/main/java/org/thingsboard/server/common/data/oauth2/ClientRegistrationDto.java index 9676d55f5f..b3b1911523 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Client.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/ClientRegistrationDto.java @@ -13,27 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.oauth2; +package org.thingsboard.server.common.data.oauth2; -import lombok.Data; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.*; +import org.thingsboard.server.common.data.id.OAuth2ClientRegistrationInfoId; -@Data -public class OAuth2Client { +import java.util.List; - private String loginButtonLabel; - private String loginButtonIcon; - private String clientName; +@EqualsAndHashCode +@Data +@ToString(exclude = {"clientSecret"}) +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ClientRegistrationDto { + private OAuth2MapperConfig mapperConfig; private String clientId; private String clientSecret; - private String accessTokenUri; private String authorizationUri; - private String scope; - private String redirectUriTemplate; - private String jwkSetUri; - private String authorizationGrantType; - private String clientAuthenticationMethod; + private String accessTokenUri; + private List scope; private String userInfoUri; private String userNameAttributeName; - private OAuth2ClientMapperConfig mapperConfig; - + private String jwkSetUri; + private String clientAuthenticationMethod; + private String loginButtonLabel; + private String loginButtonIcon; + private JsonNode additionalInfo; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/DomainInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/DomainInfo.java new file mode 100644 index 0000000000..7eb5b34661 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/DomainInfo.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import lombok.*; + +@EqualsAndHashCode +@Data +@ToString +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class DomainInfo { + private SchemeType scheme; + private String name; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/ExtendedOAuth2ClientRegistrationInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/ExtendedOAuth2ClientRegistrationInfo.java new file mode 100644 index 0000000000..9998b2bd35 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/ExtendedOAuth2ClientRegistrationInfo.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode(callSuper = true) +@Data +public class ExtendedOAuth2ClientRegistrationInfo extends OAuth2ClientRegistrationInfo { + + private String domainName; + private SchemeType domainScheme; + + public ExtendedOAuth2ClientRegistrationInfo() { + super(); + } + + public ExtendedOAuth2ClientRegistrationInfo(OAuth2ClientRegistrationInfo oAuth2ClientRegistrationInfo, + SchemeType domainScheme, + String domainName) { + super(oAuth2ClientRegistrationInfo); + this.domainScheme = domainScheme; + this.domainName = domainName; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/MapperType.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/MapperType.java new file mode 100644 index 0000000000..25faad187e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/MapperType.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +public enum MapperType { + BASIC, CUSTOM, GITHUB; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2BasicMapperConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2BasicMapperConfig.java new file mode 100644 index 0000000000..3f9f1c6dc2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2BasicMapperConfig.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import lombok.*; + +@Builder(toBuilder = true) +@EqualsAndHashCode +@Data +@ToString +public class OAuth2BasicMapperConfig { + private final String emailAttributeKey; + private final String firstNameAttributeKey; + private final String lastNameAttributeKey; + private final TenantNameStrategyType tenantNameStrategy; + private final String tenantNamePattern; + private final String customerNamePattern; + private final String defaultDashboardName; + private final boolean alwaysFullScreen; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java index 0ee5832e63..f15706a55e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java @@ -17,27 +17,20 @@ package org.thingsboard.server.common.data.oauth2; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.BaseData; -import org.thingsboard.server.common.data.id.OAuth2IntegrationId; +import lombok.NoArgsConstructor; +import lombok.AllArgsConstructor; -@EqualsAndHashCode(callSuper = true) +@EqualsAndHashCode @Data -public class OAuth2ClientInfo extends BaseData { +@NoArgsConstructor +@AllArgsConstructor +public class OAuth2ClientInfo { private String name; private String icon; private String url; - public OAuth2ClientInfo() { - super(); - } - - public OAuth2ClientInfo(OAuth2IntegrationId id) { - super(id); - } - public OAuth2ClientInfo(OAuth2ClientInfo oauth2ClientInfo) { - super(oauth2ClientInfo); this.name = oauth2ClientInfo.getName(); this.icon = oauth2ClientInfo.getIcon(); this.url = oauth2ClientInfo.getUrl(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientRegistration.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientRegistration.java new file mode 100644 index 0000000000..0ced05cfdb --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientRegistration.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.id.OAuth2ClientRegistrationId; +import org.thingsboard.server.common.data.id.OAuth2ClientRegistrationInfoId; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +@NoArgsConstructor +public class OAuth2ClientRegistration extends BaseData { + + private OAuth2ClientRegistrationInfoId clientRegistrationId; + private String domainName; + private SchemeType domainScheme; + + public OAuth2ClientRegistration(OAuth2ClientRegistration clientRegistration) { + super(clientRegistration); + this.clientRegistrationId = clientRegistration.clientRegistrationId; + this.domainName = clientRegistration.domainName; + this.domainScheme = clientRegistration.domainScheme; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientRegistrationInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientRegistrationInfo.java new file mode 100644 index 0000000000..177d677abf --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientRegistrationInfo.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; +import org.thingsboard.server.common.data.id.OAuth2ClientRegistrationInfoId; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString(exclude = {"clientSecret"}) +@NoArgsConstructor +public class OAuth2ClientRegistrationInfo extends SearchTextBasedWithAdditionalInfo implements HasName { + + private boolean enabled; + private OAuth2MapperConfig mapperConfig; + private String clientId; + private String clientSecret; + private String authorizationUri; + private String accessTokenUri; + private List scope; + private String userInfoUri; + private String userNameAttributeName; + private String jwkSetUri; + private String clientAuthenticationMethod; + private String loginButtonLabel; + private String loginButtonIcon; + + public OAuth2ClientRegistrationInfo(OAuth2ClientRegistrationInfo clientRegistration) { + super(clientRegistration); + this.enabled = clientRegistration.enabled; + this.mapperConfig = clientRegistration.mapperConfig; + this.clientId = clientRegistration.clientId; + this.clientSecret = clientRegistration.clientSecret; + this.authorizationUri = clientRegistration.authorizationUri; + this.accessTokenUri = clientRegistration.accessTokenUri; + this.scope = clientRegistration.scope; + this.userInfoUri = clientRegistration.userInfoUri; + this.userNameAttributeName = clientRegistration.userNameAttributeName; + this.jwkSetUri = clientRegistration.jwkSetUri; + this.clientAuthenticationMethod = clientRegistration.clientAuthenticationMethod; + this.loginButtonLabel = clientRegistration.loginButtonLabel; + this.loginButtonIcon = clientRegistration.loginButtonIcon; + } + + @Override + @JsonProperty(access = JsonProperty.Access.READ_ONLY) + public String getName() { + return loginButtonLabel; + } + + @Override + public String getSearchText() { + return getName(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientRegistrationTemplate.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientRegistrationTemplate.java new file mode 100644 index 0000000000..30c6e1c4a9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientRegistrationTemplate.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; +import org.thingsboard.server.common.data.id.OAuth2ClientRegistrationTemplateId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +@NoArgsConstructor +public class OAuth2ClientRegistrationTemplate extends SearchTextBasedWithAdditionalInfo implements HasName { + + private String providerId; + private OAuth2MapperConfig mapperConfig; + private String authorizationUri; + private String accessTokenUri; + private List scope; + private String userInfoUri; + private String userNameAttributeName; + private String jwkSetUri; + private String clientAuthenticationMethod; + private String comment; + private String loginButtonIcon; + private String loginButtonLabel; + private String helpLink; + + public OAuth2ClientRegistrationTemplate(OAuth2ClientRegistrationTemplate clientRegistrationTemplate) { + super(clientRegistrationTemplate); + this.providerId = clientRegistrationTemplate.providerId; + this.mapperConfig = clientRegistrationTemplate.mapperConfig; + this.authorizationUri = clientRegistrationTemplate.authorizationUri; + this.accessTokenUri = clientRegistrationTemplate.accessTokenUri; + this.scope = clientRegistrationTemplate.scope; + this.userInfoUri = clientRegistrationTemplate.userInfoUri; + this.userNameAttributeName = clientRegistrationTemplate.userNameAttributeName; + this.jwkSetUri = clientRegistrationTemplate.jwkSetUri; + this.clientAuthenticationMethod = clientRegistrationTemplate.clientAuthenticationMethod; + this.comment = clientRegistrationTemplate.comment; + this.loginButtonIcon = clientRegistrationTemplate.loginButtonIcon; + this.loginButtonLabel = clientRegistrationTemplate.loginButtonLabel; + this.helpLink = clientRegistrationTemplate.helpLink; + } + + @Override + public String getName() { + return providerId; + } + + @Override + public String getSearchText() { + return getName(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientsDomainParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientsDomainParams.java new file mode 100644 index 0000000000..4e932fad85 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientsDomainParams.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import lombok.*; + +import java.util.List; +import java.util.Set; + +@EqualsAndHashCode +@Data +@ToString +@Builder(toBuilder = true) +@NoArgsConstructor +@AllArgsConstructor +public class OAuth2ClientsDomainParams { + private List domainInfos; + private List clientRegistrations; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientsParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientsParams.java new file mode 100644 index 0000000000..ad5d5305bd --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientsParams.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import lombok.*; + +import java.util.List; +import java.util.Set; + +@EqualsAndHashCode +@Data +@ToString +@Builder(toBuilder = true) +@NoArgsConstructor +@AllArgsConstructor +public class OAuth2ClientsParams { + private boolean enabled; + private List domainsParams; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2CustomMapperConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2CustomMapperConfig.java new file mode 100644 index 0000000000..cead19c39c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2CustomMapperConfig.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import lombok.*; + +@Builder(toBuilder = true) +@EqualsAndHashCode +@Data +@ToString(exclude = {"password"}) +public class OAuth2CustomMapperConfig { + private final String url; + private final String username; + private final String password; + private final boolean sendToken; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2MapperConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2MapperConfig.java new file mode 100644 index 0000000000..15b3067cda --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2MapperConfig.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@Builder(toBuilder = true) +@EqualsAndHashCode +@Data +@ToString +public class OAuth2MapperConfig { + private boolean allowUserCreation; + private boolean activateUser; + private MapperType type; + private OAuth2BasicMapperConfig basic; + private OAuth2CustomMapperConfig custom; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/SchemeType.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/SchemeType.java new file mode 100644 index 0000000000..80d0362b7d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/SchemeType.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +public enum SchemeType { + HTTP, HTTPS, MIXED; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/TenantNameStrategyType.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/TenantNameStrategyType.java new file mode 100644 index 0000000000..91562a883a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/TenantNameStrategyType.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +public enum TenantNameStrategyType { + DOMAIN, EMAIL, CUSTOM; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java new file mode 100644 index 0000000000..436aa44230 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.query; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +public class ApiUsageStateFilter implements EntityFilter { + @Override + public EntityFilterType getType() { + return EntityFilterType.API_USAGE_STATE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValueSourceType.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValueSourceType.java index 96734fde92..7da9ab00af 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValueSourceType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValueSourceType.java @@ -18,5 +18,6 @@ package org.thingsboard.server.common.data.query; public enum DynamicValueSourceType { CURRENT_TENANT, CURRENT_CUSTOMER, - CURRENT_USER + CURRENT_USER, + CURRENT_DEVICE } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java index d3b9c39530..f2a6dde626 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java @@ -32,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = AssetTypeFilter.class, name = "assetType"), @JsonSubTypes.Type(value = DeviceTypeFilter.class, name = "deviceType"), @JsonSubTypes.Type(value = EntityViewTypeFilter.class, name = "entityViewType"), + @JsonSubTypes.Type(value = ApiUsageStateFilter.class, name = "apiUsageState"), @JsonSubTypes.Type(value = RelationsQueryFilter.class, name = "relationsQuery"), @JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"), @JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"), diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java index 0c05fdc8e4..12860b8453 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java @@ -25,7 +25,8 @@ public enum EntityFilterType { RELATIONS_QUERY("relationsQuery"), ASSET_SEARCH_QUERY("assetSearchQuery"), DEVICE_SEARCH_QUERY("deviceSearchQuery"), - ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"); + ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"), + API_USAGE_STATE("apiUsageState"); private final String label; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/AwsSnsSmsProviderConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/AwsSnsSmsProviderConfiguration.java new file mode 100644 index 0000000000..2acad17a0f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/AwsSnsSmsProviderConfiguration.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sms.config; + +import lombok.Data; + +@Data +public class AwsSnsSmsProviderConfiguration implements SmsProviderConfiguration { + + private String accessKeyId; + private String secretAccessKey; + private String region; + + @Override + public SmsProviderType getType() { + return SmsProviderType.AWS_SNS; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/SmsProviderConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/SmsProviderConfiguration.java new file mode 100644 index 0000000000..8b1984e9b4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/SmsProviderConfiguration.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sms.config; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = AwsSnsSmsProviderConfiguration.class, name = "AWS_SNS"), + @JsonSubTypes.Type(value = TwilioSmsProviderConfiguration.class, name = "TWILIO")}) +public interface SmsProviderConfiguration { + + @JsonIgnore + SmsProviderType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/SmsProviderType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/SmsProviderType.java new file mode 100644 index 0000000000..f60d917cf1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/SmsProviderType.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sms.config; + +public enum SmsProviderType { + AWS_SNS, + TWILIO +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/TestSmsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/TestSmsRequest.java new file mode 100644 index 0000000000..23c44be48c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/TestSmsRequest.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sms.config; + +import lombok.Data; + +@Data +public class TestSmsRequest { + + private SmsProviderConfiguration providerConfiguration; + private String numberTo; + private String message; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/TwilioSmsProviderConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/TwilioSmsProviderConfiguration.java new file mode 100644 index 0000000000..6f9b226711 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sms/config/TwilioSmsProviderConfiguration.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sms.config; + +import lombok.Data; + +@Data +public class TwilioSmsProviderConfiguration implements SmsProviderConfiguration { + + private String accountSid; + private String accountToken; + private String numberFrom; + + @Override + public SmsProviderType getType() { + return SmsProviderType.TWILIO; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java new file mode 100644 index 0000000000..054f78766b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -0,0 +1,87 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.tenant.profile; + +import lombok.Data; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.TenantProfileType; + +@Data +public class DefaultTenantProfileConfiguration implements TenantProfileConfiguration { + + private long maxDevices; + private long maxAssets; + private long maxCustomers; + private long maxUsers; + private long maxDashboards; + private long maxRuleChains; + + private String transportTenantMsgRateLimit; + private String transportTenantTelemetryMsgRateLimit; + private String transportTenantTelemetryDataPointsRateLimit; + private String transportDeviceMsgRateLimit; + private String transportDeviceTelemetryMsgRateLimit; + private String transportDeviceTelemetryDataPointsRateLimit; + + private long maxTransportMessages; + private long maxTransportDataPoints; + private long maxREExecutions; + private long maxJSExecutions; + private long maxDPStorageDays; + private int maxRuleNodeExecutionsPerMessage; + private long maxEmails; + private long maxSms; + + private int defaultStorageTtlDays; + + private double warnThreshold; + + @Override + public long getProfileThreshold(ApiUsageRecordKey key) { + switch (key) { + case TRANSPORT_MSG_COUNT: + return maxTransportMessages; + case TRANSPORT_DP_COUNT: + return maxTransportDataPoints; + case JS_EXEC_COUNT: + return maxJSExecutions; + case RE_EXEC_COUNT: + return maxREExecutions; + case STORAGE_DP_COUNT: + return maxDPStorageDays; + case EMAIL_EXEC_COUNT: + return maxEmails; + case SMS_EXEC_COUNT: + return maxSms; + } + return 0L; + } + + @Override + public long getWarnThreshold(ApiUsageRecordKey key) { + return (long) (getProfileThreshold(key) * (warnThreshold > 0.0 ? warnThreshold : 0.8)); + } + + @Override + public TenantProfileType getType() { + return TenantProfileType.DEFAULT; + } + + @Override + public int getMaxRuleNodeExecsPerMessage() { + return maxRuleNodeExecutionsPerMessage; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java new file mode 100644 index 0000000000..ea4a0e1fbe --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.tenant.profile; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.TenantProfileType; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = DefaultTenantProfileConfiguration.class, name = "DEFAULT")}) +public interface TenantProfileConfiguration { + + @JsonIgnore + TenantProfileType getType(); + + @JsonIgnore + long getProfileThreshold(ApiUsageRecordKey key); + + @JsonIgnore + long getWarnThreshold(ApiUsageRecordKey key); + + @JsonIgnore + int getMaxRuleNodeExecsPerMessage(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java new file mode 100644 index 0000000000..297e562d26 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.tenant.profile; + +import lombok.Data; + +@Data +public class TenantProfileData { + + private TenantProfileConfiguration configuration; + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index f0af529be9..0b257d9d6e 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -18,27 +18,27 @@ package org.thingsboard.server.common.msg; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; +import lombok.AccessLevel; import lombok.Builder; import lombok.Data; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import java.io.IOException; import java.io.Serializable; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; /** * Created by ashvayka on 13.01.18. */ @Data -@Builder @Slf4j public final class TbMsg implements Serializable { @@ -52,51 +52,73 @@ public final class TbMsg implements Serializable { private final String data; private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; + @Getter(value = AccessLevel.NONE) + private final AtomicInteger ruleNodeExecCounter; + + public int getAndIncrementRuleNodeCounter() { + return ruleNodeExecCounter.getAndIncrement(); + } + //This field is not serialized because we use queues and there is no need to do it @JsonIgnore transient private final TbMsgCallback callback; public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, + metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY); } public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY); } + // REALLY NEW MSG + public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); + return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY); } public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, 0, TbMsgCallback.EMPTY); } + // For Tests only + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY); } public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, callback); + } + + public static TbMsg transformMsg(TbMsg tbMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { + return new TbMsg(tbMsg.getQueueName(), tbMsg.getId(), tbMsg.getTs(), type, originator, metaData.copy(), tbMsg.getDataType(), + data, tbMsg.getRuleChainId(), tbMsg.getRuleNodeId(), tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback()); + } + + public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId) { + return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType, + tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback()); } - public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getQueueName(), origMsg.getId(), origMsg.getTs(), type, originator, metaData.copy(), origMsg.getDataType(), - data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); + public static TbMsg transformMsg(TbMsg tbMsg, String queueName) { + return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType, + tbMsg.data, tbMsg.getRuleChainId(), null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback()); } - public static TbMsg transformMsg(TbMsg origMsg, RuleChainId ruleChainId) { - return new TbMsg(origMsg.queueName, origMsg.id, origMsg.ts, origMsg.type, origMsg.originator, origMsg.metaData, origMsg.dataType, - origMsg.data, ruleChainId, null, origMsg.getCallback()); + public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId, String queueName) { + return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType, + tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback()); } public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { return new TbMsg(tbMsg.getQueueName(), UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), - tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, tbMsg.ruleNodeExecCounter.get(), TbMsgCallback.EMPTY); } private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, int ruleNodeExecCounter, TbMsgCallback callback) { this.id = id; this.queueName = queueName; if (ts > 0) { @@ -111,6 +133,7 @@ public final class TbMsg implements Serializable { this.data = data; this.ruleChainId = ruleChainId; this.ruleNodeId = ruleNodeId; + this.ruleNodeExecCounter = new AtomicInteger(ruleNodeExecCounter); if (callback != null) { this.callback = callback; } else { @@ -147,6 +170,7 @@ public final class TbMsg implements Serializable { builder.setDataType(msg.getDataType().ordinal()); builder.setData(msg.getData()); + builder.setRuleNodeExecCounter(msg.ruleNodeExecCounter.get()); return builder.build().toByteArray(); } @@ -164,18 +188,18 @@ public final class TbMsg implements Serializable { ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); } TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); + return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, proto.getRuleNodeExecCounter(), callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } } public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { - return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); + return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, this.ruleNodeExecCounter.get(), callback); } public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); + return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, this.ruleNodeExecCounter.get(), callback); } public TbMsgCallback getCallback() { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index bf0fffe404..c2dec02612 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -50,6 +50,10 @@ public class TopicPartitionInfo { this.fullTopicName = tmp; } + public TopicPartitionInfo newByTopic(String topic) { + return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.myPartition); + } + public String getTopic() { return topic; } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/session/FeatureType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/session/FeatureType.java index 9f421c0e73..58289cc94b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/session/FeatureType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/session/FeatureType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.common.msg.session; public enum FeatureType { - ATTRIBUTES, TELEMETRY, RPC, CLAIM + ATTRIBUTES, TELEMETRY, RPC, CLAIM, PROVISION } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java index fea6fcb337..57cc578f2c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java @@ -15,10 +15,19 @@ */ package org.thingsboard.server.common.msg.tools; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAdjuster; +import java.time.temporal.TemporalAdjusters; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoField.DAY_OF_MONTH; +import static java.time.temporal.ChronoUnit.MONTHS; + public class SchedulerUtils { private static final ConcurrentMap tzMap = new ConcurrentHashMap<>(); @@ -27,4 +36,40 @@ public class SchedulerUtils { return tzMap.computeIfAbsent(tz == null || tz.isEmpty() ? "UTC" : tz, ZoneId::of); } + public static long getStartOfCurrentHour() { + return getStartOfCurrentHour(UTC); + } + + public static long getStartOfCurrentHour(ZoneId zoneId) { + return LocalDateTime.now(UTC).atZone(zoneId).truncatedTo(ChronoUnit.HOURS).toInstant().toEpochMilli(); + } + + public static long getStartOfCurrentMonth() { + return getStartOfCurrentMonth(UTC); + } + + public static long getStartOfCurrentMonth(ZoneId zoneId) { + return LocalDate.now(UTC).withDayOfMonth(1).atStartOfDay(zoneId).toInstant().toEpochMilli(); + } + + public static long getStartOfNextMonth() { + return getStartOfNextMonth(UTC); + } + + public static long getStartOfNextMonth(ZoneId zoneId) { + return LocalDate.now(UTC).with(TemporalAdjusters.firstDayOfNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli(); + } + + public static long getStartOfNextNextMonth() { + return getStartOfNextNextMonth(UTC); + } + + public static long getStartOfNextNextMonth(ZoneId zoneId) { + return LocalDate.now(UTC).with(firstDayOfNextNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli(); + } + + public static TemporalAdjuster firstDayOfNextNextMonth() { + return (temporal) -> temporal.with(DAY_OF_MONTH, 1).plus(2, MONTHS); + } + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java index 3c79895c8b..3550bc44a9 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java @@ -50,4 +50,8 @@ public class TbRateLimits { return bucket.tryConsume(1); } + public boolean tryConsume(long number) { + return bucket.tryConsume(number); + } + } diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 25e006121f..53cb90014b 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -45,4 +45,5 @@ message TbMsgProto { string data = 14; int64 ts = 15; + int32 ruleNodeExecCounter = 16; } \ No newline at end of file diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java new file mode 100644 index 0000000000..c28f3637b7 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; + +import java.util.concurrent.atomic.AtomicInteger; + +public class MultipleTbQueueCallbackWrapper implements TbQueueCallback { + + private final AtomicInteger tbQueueCallbackCount; + private final TbQueueCallback callback; + + public MultipleTbQueueCallbackWrapper(int tbQueueCallbackCount, TbQueueCallback callback) { + this.tbQueueCallbackCount = new AtomicInteger(tbQueueCallbackCount); + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (tbQueueCallbackCount.decrementAndGet() <= 0) { + callback.onSuccess(metadata); + } + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(new RuleEngineException(t.getMessage())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java index 9017823efb..6a54feedd5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java @@ -20,6 +20,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.annotation.DependsOn; import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import java.util.Collections; @@ -40,6 +41,7 @@ public class DummyDiscoveryService implements DiscoveryService { } @EventListener(ApplicationReadyEvent.class) + @Order(value = 1) public void onApplicationEvent(ApplicationReadyEvent event) { partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList()); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index c683ed846d..5cceb43324 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -195,9 +195,11 @@ public class HashPartitionService implements PartitionService { if (current.getServiceTypesList().contains(serviceType.name())) { result.add(current.getServiceId()); } - for (ServiceInfo serviceInfo : currentOtherServices) { - if (serviceInfo.getServiceTypesList().contains(serviceType.name())) { - result.add(serviceInfo.getServiceId()); + if (currentOtherServices != null) { + for (ServiceInfo serviceInfo : currentOtherServices) { + if (serviceInfo.getServiceTypesList().contains(serviceType.name())) { + result.add(serviceInfo.getServiceId()); + } } } return result; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index e761a229c4..684c33f5a7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -34,6 +34,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; @@ -112,6 +113,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } @EventListener(ApplicationReadyEvent.class) + @Order(value = 1) public void onApplicationEvent(ApplicationReadyEvent event) { if (stopped) { log.debug("Ignoring application ready event. Service is stopped."); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 1fbfb68e3e..90b560d0fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -46,7 +46,6 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, boolean autoCommit, int autoCommitIntervalMs, - int maxPollRecords, TbQueueAdmin admin) { super(topic); Properties props = settings.toProps(); @@ -54,6 +53,9 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue if (groupId != null) { props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); } + if (settings.getMaxPollIntervalMs() > 0) { + props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, settings.getMaxPollIntervalMs()); + } props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, settings.getMaxPollRecords()); props.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, settings.getMaxPartitionFetchBytes()); props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, settings.getFetchMaxBytes()); @@ -61,9 +63,6 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - if (maxPollRecords > 0) { - props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); - } this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 37e978c07d..15057d64d3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -63,6 +63,10 @@ public class TbKafkaSettings { @Getter private int maxPollRecords; + @Value("${queue.kafka.max_poll_interval_ms:0}") + @Getter + private int maxPollIntervalMs; + @Value("${queue.kafka.max_partition_fetch_bytes:16777216}") @Getter private int maxPartitionFetchBytes; @@ -111,4 +115,5 @@ public class TbKafkaSettings { } return props; } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index c1d724c207..0299720005 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -22,7 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -91,64 +91,64 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> createTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); } @@ -175,6 +175,17 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 4889a7182d..4625b9d4fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -163,6 +164,17 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index cf43bc5fe4..851b8cc40e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -143,6 +143,11 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -158,4 +163,5 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory notificationAdmin.destroy(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 8106ee084b..103711f36b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -109,6 +110,11 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 8ebda6e7b9..f7a47ecf3a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -125,8 +125,20 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new InMemoryTbQueueConsumer<>(coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic()); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); } + + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 9e8e326f0d..9ddf72534c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -96,4 +97,10 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory public TbQueueConsumer> createTransportNotificationsConsumer() { return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic()); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 31ef4d3e45..538d91aa0f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -249,6 +250,28 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getUsageStatsTopic()); + consumerBuilder.clientId("monolith-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index bf2ec6751e..5e1b398e91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -219,6 +220,28 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getUsageStatsTopic()); + consumerBuilder.clientId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index dc44caf3f0..118f0f3316 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -192,6 +193,16 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 1cf8bca4b2..7961bb97bb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -138,6 +139,16 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { return responseBuilder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-node-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 29a60af1e2..d7c7542626 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -27,6 +27,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -183,6 +184,17 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 7d85b5c3ba..71a1ceaf9b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -157,6 +158,17 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index a6105fea0c..ad679039e4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -146,6 +147,11 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index 72665c5163..efde055271 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -114,6 +115,11 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 1f812ea13c..e6eff652be 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -22,7 +22,14 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -91,64 +98,64 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> createTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); } @@ -175,6 +182,17 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index 65d582c4bc..e3064dd62d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -163,6 +164,17 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index 26abf78164..a1097049db 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -144,6 +145,11 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java index 03cd6eb557..b28a2cb7e2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -111,6 +112,11 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 3db6496b7d..a205f9b061 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -180,6 +181,17 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index a3ba577f06..f00aee331b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -21,10 +21,12 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -101,7 +103,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @@ -111,7 +113,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @@ -122,10 +124,10 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override @@ -162,6 +164,17 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index e4f6dbfbb4..40a183a254 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -21,10 +21,12 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -93,7 +95,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @@ -103,7 +105,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @@ -114,10 +116,10 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override @@ -143,6 +145,11 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java index 1e900f37e5..4781d46583 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -18,19 +18,24 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @@ -69,17 +74,17 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producerTemplate = + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbQueueProducer> producerTemplate = new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); - TbQueueConsumer> consumerTemplate = + TbQueueConsumer> consumerTemplate = new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); @@ -90,20 +95,25 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 59a8f449b0..7c19a1ba2f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -33,7 +34,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbCoreQueueFactory { +public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service @@ -77,6 +78,13 @@ public interface TbCoreQueueFactory { */ TbQueueConsumer> createToCoreMsgConsumer(); + /** + * Used to consume messages about usage statistics by TB Core Service + * + * @return + */ + TbQueueConsumer> createToUsageStatsServiceMsgConsumer(); + /** * Used to consume high priority messages by TB Core Service * diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 9fe5de0bcc..e620259960 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -28,11 +33,12 @@ import javax.annotation.PostConstruct; public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private final TbCoreQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - private TbQueueProducer> toRuleEngineNotifications; - private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toUsageStats; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -45,30 +51,36 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { return toTransport; } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> getRuleEngineMsgProducer() { return toRuleEngine; } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { return toRuleEngineNotifications; } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> getTbCoreMsgProducer() { return toTbCore; } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { return toTbCoreNotifications; } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 34c6ee577f..8c3a0b6565 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -20,6 +20,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -63,4 +64,10 @@ public interface TbQueueProducerProvider { */ TbQueueProducer> getTbCoreNotificationsMsgProducer(); + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbUsageStatsMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 69de7ca3e2..309def1269 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -28,12 +33,12 @@ import javax.annotation.PostConstruct; public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private final TbRuleEngineQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - private TbQueueProducer> toRuleEngineNotifications; - private TbQueueProducer> toTbCoreNotifications; - + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toUsageStats; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -46,30 +51,36 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { return toTransport; } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> getRuleEngineMsgProducer() { return toRuleEngine; } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { return toRuleEngineNotifications; } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> getTbCoreMsgProducer() { return toTbCore; } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { return toTbCoreNotifications; } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 561b3e84ea..2f80bbce45 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -32,7 +32,7 @@ import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbRuleEngineQueueFactory { +public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java index dc1d2c449c..bd6c6e0584 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java @@ -15,17 +15,17 @@ */ package org.thingsboard.server.queue.provider; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; -public interface TbTransportQueueFactory { +public interface TbTransportQueueFactory extends TbUsageStatsClientQueueFactory { TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index bced3f83ac..dee6508ef4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -28,8 +33,9 @@ import javax.annotation.PostConstruct; public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { private final TbTransportQueueFactory tbQueueProvider; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toUsageStats; public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -39,30 +45,36 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider public void init() { this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> getRuleEngineMsgProducer() { return toRuleEngine; } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> getTbCoreMsgProducer() { return toTbCore; } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java new file mode 100644 index 0000000000..f2ce1a9fe5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface TbUsageStatsClientQueueFactory { + + TbQueueProducer> createToUsageStatsServiceMsgProducer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java new file mode 100644 index 0000000000..02ce60496e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.scheduler; + +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardThreadFactory; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +@Component +public class DefaultSchedulerComponent implements SchedulerComponent{ + + protected ScheduledExecutorService schedulerExecutor; + + @PostConstruct + public void init(){ + this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("queue-scheduler")); + } + + @PreDestroy + public void destroy() { + if (schedulerExecutor != null) { + schedulerExecutor.shutdownNow(); + } + } + + public ScheduledFuture> schedule(Runnable command, long delay, TimeUnit unit) { + return schedulerExecutor.schedule(command, delay, unit); + } + + public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { + return schedulerExecutor.schedule(callable, delay, unit); + } + + public ScheduledFuture> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) { + return schedulerExecutor.scheduleAtFixedRate(command, initialDelay, period, unit); + } + + public ScheduledFuture> scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) { + return schedulerExecutor.scheduleWithFixedDelay(command, initialDelay, delay, unit); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/SchedulerComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/SchedulerComponent.java new file mode 100644 index 0000000000..3c12be43a5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/SchedulerComponent.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.scheduler; + +import java.util.concurrent.Callable; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public interface SchedulerComponent { + + ScheduledFuture> schedule(Runnable command, long delay, TimeUnit unit); + + ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit); + + ScheduledFuture> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit); + + ScheduledFuture> scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 1c260c754d..d78c6d1bad 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -26,6 +26,9 @@ public class TbQueueCoreSettings { @Value("${queue.core.topic}") private String topic; + @Value("${queue.core.usage-stats-topic:tb_usage_stats}") + private String usageStatsTopic; + @Value("${queue.core.partitions}") private int partitions; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index 3e508b09e6..a4e977b6f0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -87,8 +87,9 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg))); - sendMsgRequest.withMessageGroupId(tpi.getTopic()); - sendMsgRequest.withMessageDeduplicationId(UUID.randomUUID().toString()); + String sqsMsgId = UUID.randomUUID().toString(); + sendMsgRequest.withMessageGroupId(sqsMsgId); + sendMsgRequest.withMessageDeduplicationId(sqsMsgId); ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageClient.java new file mode 100644 index 0000000000..5c5b593d96 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageClient.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.usagestats; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; + +import javax.annotation.PostConstruct; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +@Component +@Slf4j +public class DefaultTbApiUsageClient implements TbApiUsageClient { + + @Value("${usage.stats.report.enabled:true}") + private boolean enabled; + @Value("${usage.stats.report.interval:10}") + private int interval; + + private final ConcurrentMap[] values = new ConcurrentMap[ApiUsageRecordKey.values().length]; + private final PartitionService partitionService; + private final SchedulerComponent scheduler; + private final TbQueueProducerProvider producerProvider; + private TbQueueProducer> msgProducer; + + public DefaultTbApiUsageClient(PartitionService partitionService, SchedulerComponent scheduler, TbQueueProducerProvider producerProvider) { + this.partitionService = partitionService; + this.scheduler = scheduler; + this.producerProvider = producerProvider; + } + + @PostConstruct + private void init() { + if (enabled) { + msgProducer = this.producerProvider.getTbUsageStatsMsgProducer(); + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + values[key.ordinal()] = new ConcurrentHashMap<>(); + } + scheduler.scheduleWithFixedDelay(this::reportStats, new Random().nextInt(interval), interval, TimeUnit.SECONDS); + } + } + + private void reportStats() { + try { + ConcurrentMap report = new ConcurrentHashMap<>(); + + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + values[key.ordinal()].forEach(((tenantId, atomicLong) -> { + long value = atomicLong.getAndSet(0); + if (value > 0) { + ToUsageStatsServiceMsg.Builder msgBuilder = report.computeIfAbsent(tenantId, id -> { + ToUsageStatsServiceMsg.Builder msg = ToUsageStatsServiceMsg.newBuilder(); + msg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + msg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + return msg; + }); + msgBuilder.addValues(UsageStatsKVProto.newBuilder().setKey(key.name()).setValue(value).build()); + } + })); + } + + report.forEach(((tenantId, builder) -> { + //TODO: figure out how to minimize messages into the queue. Maybe group by 100s of messages? + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).newByTopic(msgProducer.getDefaultTopic()); + msgProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), builder.build()), null); + })); + if (!report.isEmpty()) { + log.info("Report statistics for: {} tenants", report.size()); + } + } catch (Exception e) { + log.warn("Failed to report statistics: ", e); + } + } + + @Override + public void report(TenantId tenantId, ApiUsageRecordKey key, long value) { + if (enabled) { + ConcurrentMap map = values[key.ordinal()]; + AtomicLong atomicValue = map.computeIfAbsent(tenantId, id -> new AtomicLong()); + atomicValue.addAndGet(value); + } + } + + @Override + public void report(TenantId tenantId, ApiUsageRecordKey key) { + report(tenantId, key, 1L); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbApiUsageClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbApiUsageClient.java new file mode 100644 index 0000000000..31d85b27c7 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbApiUsageClient.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.usagestats; + +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.id.TenantId; + +public interface TbApiUsageClient { + + void report(TenantId tenantId, ApiUsageRecordKey key, long value); + + void report(TenantId tenantId, ApiUsageRecordKey key); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbTransportComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbTransportComponent.java new file mode 100644 index 0000000000..dde1c6a620 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbTransportComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport'") +public @interface TbTransportComponent { +} diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 9a4b422007..29b77348c8 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -73,6 +73,12 @@ enum KeyValueType { JSON_V = 4; } +enum CredentialsType { + ACCESS_TOKEN = 0; + X509_CERTIFICATE = 1; + MQTT_BASIC = 2; +} + message KeyValueProto { string key = 1; KeyValueType type = 2; @@ -171,32 +177,27 @@ message GetOrCreateDeviceFromGatewayResponseMsg { bytes profileBody = 2; } -message GetTenantRoutingInfoRequestMsg { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; -} - -message GetTenantRoutingInfoResponseMsg { - bool isolatedTbCore = 1; - bool isolatedTbRuleEngine = 2; -} - -message GetDeviceProfileRequestMsg { - int64 profileIdMSB = 1; - int64 profileIdLSB = 2; +message GetEntityProfileRequestMsg { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; } -message GetDeviceProfileResponseMsg { - bytes data = 1; +message GetEntityProfileResponseMsg { + string entityType = 1; + bytes data = 2; + bytes apiState = 3; } -message DeviceProfileUpdateMsg { - bytes data = 1; +message EntityUpdateMsg { + string entityType = 1; + bytes data = 2; } -message DeviceProfileDeleteMsg { - int64 profileIdMSB = 1; - int64 profileIdLSB = 2; +message EntityDeleteMsg { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; } message SessionCloseNotificationProto { @@ -241,6 +242,44 @@ message ClaimDeviceMsg { int64 durationMs = 4; } +message DeviceCredentialsProto { + int64 deviceIdMSB = 1; + int64 deviceIdLSB = 2; + CredentialsType credentialsType = 3; + string credentialsId = 4; + string credentialsValue = 5; +} + +message CredentialsDataProto { + ValidateDeviceTokenRequestMsg validateDeviceTokenRequestMsg = 1; + ValidateDeviceX509CertRequestMsg validateDeviceX509CertRequestMsg = 2; + ValidateBasicMqttCredRequestMsg validateBasicMqttCredRequestMsg = 3; +} + +message ProvisionDeviceRequestMsg { + string deviceName = 1; + CredentialsType credentialsType = 2; + ProvisionDeviceCredentialsMsg provisionDeviceCredentialsMsg = 3; + CredentialsDataProto credentialsDataProto = 4; +} + +message ProvisionDeviceCredentialsMsg { + string provisionDeviceKey = 1; + string provisionDeviceSecret = 2; +} + +message ProvisionDeviceResponseMsg { + ProvisionResponseStatus status = 1; + CredentialsType credentialsType = 2; + string credentialsValue = 3; +} + +enum ProvisionResponseStatus { + UNKNOWN = 0; + SUCCESS = 1; + NOT_FOUND = 2; + FAILURE = 3; +} //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -266,6 +305,7 @@ message TransportToDeviceActorMsg { ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 6; SubscriptionInfoProto subscriptionInfo = 7; ClaimDeviceMsg claimDevice = 8; + ProvisionDeviceRequestMsg provisionDevice = 9; } message TransportToRuleEngineMsg { @@ -453,17 +493,17 @@ message TransportApiRequestMsg { ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; - GetTenantRoutingInfoRequestMsg getTenantRoutingInfoRequestMsg = 4; - GetDeviceProfileRequestMsg getDeviceProfileRequestMsg = 5; + GetEntityProfileRequestMsg entityProfileRequestMsg = 4; ValidateBasicMqttCredRequestMsg validateBasicMqttCredRequestMsg = 6; + ProvisionDeviceRequestMsg provisionDeviceRequestMsg = 7; } /* Response from ThingsBoard Core Service to Transport Service */ message TransportApiResponseMsg { ValidateDeviceCredentialsResponseMsg validateCredResponseMsg = 1; GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; - GetTenantRoutingInfoResponseMsg getTenantRoutingInfoResponseMsg = 4; - GetDeviceProfileResponseMsg getDeviceProfileResponseMsg = 5; + GetEntityProfileResponseMsg entityProfileResponseMsg = 3; + ProvisionDeviceResponseMsg provisionDeviceResponseMsg = 4; } /* Messages that are handled by ThingsBoard Core Service */ @@ -505,6 +545,21 @@ message ToTransportMsg { AttributeUpdateNotificationMsg attributeUpdateNotification = 5; ToDeviceRpcRequestMsg toDeviceRequest = 6; ToServerRpcResponseMsg toServerResponse = 7; - DeviceProfileUpdateMsg deviceProfileUpdateMsg = 8; - DeviceProfileDeleteMsg deviceProfileDeleteMsg = 9; + /* For Tenant, TenantProfile and DeviceProfile */ + EntityUpdateMsg entityUpdateMsg = 8; + EntityDeleteMsg entityDeleteMsg = 9; + ProvisionDeviceResponseMsg provisionResponse = 10; +} + +message UsageStatsKVProto{ + string key = 1; + int64 value = 2; +} + +message ToUsageStatsServiceMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; + repeated UsageStatsKVProto values = 5; } diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index b846bac38c..63af515c00 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -24,6 +24,7 @@ import org.eclipse.californium.core.network.ExchangeObserver; import org.eclipse.californium.core.server.resources.CoapExchange; import org.eclipse.californium.core.server.resources.Resource; import org.springframework.util.ReflectionUtils; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.security.DeviceTokenCredentials; import org.thingsboard.server.common.msg.session.FeatureType; @@ -33,9 +34,11 @@ import org.thingsboard.server.common.transport.TransportContext; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; import java.lang.reflect.Field; import java.util.List; @@ -130,10 +133,25 @@ public class CoapTransportResource extends CoapResource { case CLAIM: processRequest(exchange, SessionMsgType.CLAIM_REQUEST); break; + case PROVISION: + processProvision(exchange); + break; } } } + private void processProvision(CoapExchange exchange) { + log.trace("Processing {}", exchange.advanced().getRequest()); + exchange.accept(); + try { + transportService.process(transportContext.getAdaptor().convertToProvisionRequestMsg(UUID.randomUUID(), exchange.advanced().getRequest()), + new DeviceProvisionCallback(exchange)); + } catch (AdaptorException e) { + log.trace("Failed to decode message: ", e); + exchange.respond(ResponseCode.BAD_REQUEST); + } + } + private void processRequest(CoapExchange exchange, SessionMsgType type) { log.trace("Processing {}", exchange.advanced().getRequest()); exchange.accept(); @@ -274,6 +292,8 @@ public class CoapTransportResource extends CoapResource { try { if (uriPath.size() >= FEATURE_TYPE_POSITION) { return Optional.of(FeatureType.valueOf(uriPath.get(FEATURE_TYPE_POSITION - 1).toUpperCase())); + } else if (uriPath.size() == 3 && uriPath.contains(DataConstants.PROVISION)) { + return Optional.of(FeatureType.valueOf(DataConstants.PROVISION.toUpperCase())); } } catch (RuntimeException e) { log.warn("Failed to decode feature type: {}", uriPath); @@ -325,6 +345,25 @@ public class CoapTransportResource extends CoapResource { } } + private static class DeviceProvisionCallback implements TransportServiceCallback { + private final CoapExchange exchange; + + DeviceProvisionCallback(CoapExchange exchange) { + this.exchange = exchange; + } + + @Override + public void onSuccess(TransportProtos.ProvisionDeviceResponseMsg msg) { + exchange.respond(JsonConverter.toJson(msg).toString()); + } + + @Override + public void onError(Throwable e) { + log.warn("Failed to process request", e); + exchange.respond(ResponseCode.INTERNAL_SERVER_ERROR); + } + } + private static class CoapOkCallback implements TransportServiceCallback { private final CoapExchange exchange; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index afb83f513f..a6813e488d 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.eclipse.californium.core.CoapResource; import org.eclipse.californium.core.CoapServer; import org.eclipse.californium.core.network.CoapEndpoint; +import org.eclipse.californium.core.network.config.NetworkConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; @@ -46,7 +47,7 @@ public class CoapTransportService { public void init() throws UnknownHostException { log.info("Starting CoAP transport..."); log.info("Starting CoAP transport server"); - this.server = new CoapServer(); + this.server = new CoapServer(NetworkConfig.createStandardWithoutFile()); createResources(); InetAddress addr = InetAddress.getByName(coapTransportContext.getHost()); InetSocketAddress sockAddr = new InetSocketAddress(addr, coapTransportContext.getPort()); diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapTransportAdaptor.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapTransportAdaptor.java index 82c0b80547..fd010a7141 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapTransportAdaptor.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapTransportAdaptor.java @@ -19,6 +19,7 @@ import org.eclipse.californium.core.coap.Request; import org.eclipse.californium.core.coap.Response; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; import org.thingsboard.server.transport.coap.CoapTransportResource; import java.util.UUID; @@ -45,4 +46,6 @@ public interface CoapTransportAdaptor { Response convertToPublish(CoapTransportResource.CoapSessionListener coapSessionListener, TransportProtos.ToServerRpcResponseMsg msg) throws AdaptorException; + ProvisionDeviceRequestMsg convertToProvisionRequestMsg(UUID sessionId, Request inbound) throws AdaptorException; + } diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java index 5c9c471570..28292f6c20 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java @@ -123,6 +123,16 @@ public class JsonCoapAdaptor implements CoapTransportAdaptor { return response; } + @Override + public TransportProtos.ProvisionDeviceRequestMsg convertToProvisionRequestMsg(UUID sessionId, Request inbound) throws AdaptorException { + String payload = validatePayload(sessionId, inbound, false); + try { + return JsonConverter.convertToProvisionRequestMsg(payload); + } catch (IllegalStateException | JsonSyntaxException ex) { + throw new AdaptorException(ex); + } + } + @Override public Response convertToPublish(CoapTransportResource.CoapSessionListener session, TransportProtos.GetAttributeResponseMsg msg) throws AdaptorException { if (msg.getClientAttributeListCount() == 0 && msg.getSharedAttributeListCount() == 0) { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index 404024b202..cdc2791839 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -44,6 +44,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotif import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; @@ -203,6 +204,14 @@ public class DeviceApiController { return responseWriter; } + @RequestMapping(value = "/provision", method = RequestMethod.POST) + public DeferredResult provisionDevice(@RequestBody String json, HttpServletRequest httpRequest) { + DeferredResult responseWriter = new DeferredResult<>(); + transportContext.getTransportService().process(JsonConverter.convertToProvisionRequestMsg(json), + new DeviceProvisionCallback(responseWriter)); + return responseWriter; + } + private static class DeviceAuthCallback implements TransportServiceCallback { private final TransportContext transportContext; private final DeferredResult responseWriter; @@ -230,6 +239,25 @@ public class DeviceApiController { } } + private static class DeviceProvisionCallback implements TransportServiceCallback { + private final DeferredResult responseWriter; + + DeviceProvisionCallback(DeferredResult responseWriter) { + this.responseWriter = responseWriter; + } + + @Override + public void onSuccess(ProvisionDeviceResponseMsg msg) { + responseWriter.setResult(new ResponseEntity<>(JsonConverter.toJson(msg).toString(), HttpStatus.OK)); + } + + @Override + public void onError(Throwable e) { + log.warn("Failed to process request", e); + responseWriter.setResult(new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR)); + } + } + private static class SessionCloseOnErrorCallback implements TransportServiceCallback { private final TransportService transportService; private final SessionInfoProto sessionInfo; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 6a09bdda40..0df14d4655 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -16,6 +16,7 @@ package org.thingsboard.server.transport.mqtt; import com.fasterxml.jackson.databind.JsonNode; +import com.google.gson.JsonParseException; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.codec.mqtt.MqttConnAckMessage; @@ -38,10 +39,14 @@ import io.netty.util.ReferenceCountUtil; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.GenericFutureListener; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.common.msg.EncryptionUtil; +import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; @@ -51,8 +56,10 @@ import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; import org.thingsboard.server.transport.mqtt.session.DeviceSessionCtx; import org.thingsboard.server.transport.mqtt.session.GatewaySessionHandler; @@ -65,14 +72,16 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.Date; +import java.util.concurrent.TimeUnit; import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_ACCEPTED; import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_REFUSED_NOT_AUTHORIZED; import static io.netty.handler.codec.mqtt.MqttMessageType.CONNACK; +import static io.netty.handler.codec.mqtt.MqttMessageType.CONNECT; import static io.netty.handler.codec.mqtt.MqttMessageType.PINGRESP; import static io.netty.handler.codec.mqtt.MqttMessageType.PUBACK; import static io.netty.handler.codec.mqtt.MqttMessageType.SUBACK; @@ -92,6 +101,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private final UUID sessionId; private final MqttTransportContext context; private final TransportService transportService; + private final SchedulerComponent scheduler; private final SslHandler sslHandler; private final ConcurrentMap mqttQoSMap; @@ -103,6 +113,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement this.sessionId = UUID.randomUUID(); this.context = context; this.transportService = context.getTransportService(); + this.scheduler = context.getScheduler(); this.sslHandler = sslHandler; this.mqttQoSMap = new ConcurrentHashMap<>(); this.deviceSessionCtx = new DeviceSessionCtx(sessionId, mqttQoSMap, context); @@ -113,7 +124,13 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement log.trace("[{}] Processing msg: {}", sessionId, msg); try { if (msg instanceof MqttMessage) { - processMqttMsg(ctx, (MqttMessage) msg); + MqttMessage message = (MqttMessage) msg; + if (message.decoderResult().isSuccess()) { + processMqttMsg(ctx, message); + } else { + log.error("[{}] Message processing failed: {}", sessionId, message.decoderResult().cause().getMessage()); + ctx.close(); + } } else { ctx.close(); } @@ -130,10 +147,59 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement return; } deviceSessionCtx.setChannel(ctx); + if (CONNECT.equals(msg.fixedHeader().messageType())) { + processConnect(ctx, (MqttConnectMessage) msg); + } else if (deviceSessionCtx.isProvisionOnly()) { + processProvisionSessionMsg(ctx, msg); + } else { + processRegularSessionMsg(ctx, msg); + } + } + + private void processProvisionSessionMsg(ChannelHandlerContext ctx, MqttMessage msg) { switch (msg.fixedHeader().messageType()) { - case CONNECT: - processConnect(ctx, (MqttConnectMessage) msg); + case PUBLISH: + MqttPublishMessage mqttMsg = (MqttPublishMessage) msg; + String topicName = mqttMsg.variableHeader().topicName(); + int msgId = mqttMsg.variableHeader().packetId(); + try { + if (topicName.equals(MqttTopics.DEVICE_PROVISION_REQUEST_TOPIC)) { + try { + TransportProtos.ProvisionDeviceRequestMsg provisionRequestMsg = deviceSessionCtx.getContext().getJsonMqttAdaptor().convertToProvisionRequestMsg(deviceSessionCtx, mqttMsg); + transportService.process(provisionRequestMsg, new DeviceProvisionCallback(ctx, msgId, provisionRequestMsg)); + log.trace("[{}][{}] Processing provision publish msg [{}][{}]!", sessionId, deviceSessionCtx.getDeviceId(), topicName, msgId); + } catch (Exception e) { + if (e instanceof JsonParseException || (e.getCause() != null && e.getCause() instanceof JsonParseException)) { + TransportProtos.ProvisionDeviceRequestMsg provisionRequestMsg = deviceSessionCtx.getContext().getProtoMqttAdaptor().convertToProvisionRequestMsg(deviceSessionCtx, mqttMsg); + transportService.process(provisionRequestMsg, new DeviceProvisionCallback(ctx, msgId, provisionRequestMsg)); + deviceSessionCtx.setProvisionPayloadType(TransportPayloadType.PROTOBUF); + log.trace("[{}][{}] Processing provision publish msg [{}][{}]!", sessionId, deviceSessionCtx.getDeviceId(), topicName, msgId); + } else { + throw e; + } + } + } else { + ctx.close(); + throw new RuntimeException("Unsupported topic for provisioning requests!"); + } + } catch (RuntimeException | AdaptorException e) { + log.warn("[{}] Failed to process publish msg [{}][{}]", sessionId, topicName, msgId, e); + ctx.close(); + } + break; + case PINGREQ: + ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0))); break; + case DISCONNECT: + if (checkConnected(ctx, msg)) { + processDisconnect(ctx); + } + break; + } + } + + private void processRegularSessionMsg(ChannelHandlerContext ctx, MqttMessage msg) { + switch (msg.fixedHeader().messageType()) { case PUBLISH: processPublish(ctx, (MqttPublishMessage) msg); break; @@ -169,7 +235,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement if (topicName.startsWith(MqttTopics.BASE_GATEWAY_API_TOPIC)) { if (gatewaySessionHandler != null) { - handleGatewayPublishMsg(topicName, msgId, mqttMsg); + handleGatewayPublishMsg(ctx, topicName, msgId, mqttMsg); transportService.reportActivity(deviceSessionCtx.getSessionInfo()); } } else { @@ -177,7 +243,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } } - private void handleGatewayPublishMsg(String topicName, int msgId, MqttPublishMessage mqttMsg) { + private void handleGatewayPublishMsg(ChannelHandlerContext ctx, String topicName, int msgId, MqttPublishMessage mqttMsg) { try { switch (topicName) { case MqttTopics.GATEWAY_TELEMETRY_TOPIC: @@ -201,6 +267,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement case MqttTopics.GATEWAY_DISCONNECT_TOPIC: gatewaySessionHandler.onDeviceDisconnect(mqttMsg); break; + default: + ack(ctx, msgId); } } catch (RuntimeException | AdaptorException e) { log.warn("[{}] Failed to process publish msg [{}][{}]", sessionId, topicName, msgId, e); @@ -210,12 +278,12 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private void processDevicePublish(ChannelHandlerContext ctx, MqttPublishMessage mqttMsg, String topicName, int msgId) { try { MqttTransportAdaptor payloadAdaptor = deviceSessionCtx.getPayloadAdaptor(); - if (deviceSessionCtx.isDeviceTelemetryTopic(topicName)) { - TransportProtos.PostTelemetryMsg postTelemetryMsg = payloadAdaptor.convertToPostTelemetry(deviceSessionCtx, mqttMsg); - transportService.process(deviceSessionCtx.getSessionInfo(), postTelemetryMsg, getPubAckCallback(ctx, msgId, postTelemetryMsg)); - } else if (deviceSessionCtx.isDeviceAttributesTopic(topicName)) { + if (deviceSessionCtx.isDeviceAttributesTopic(topicName)) { TransportProtos.PostAttributeMsg postAttributeMsg = payloadAdaptor.convertToPostAttributes(deviceSessionCtx, mqttMsg); transportService.process(deviceSessionCtx.getSessionInfo(), postAttributeMsg, getPubAckCallback(ctx, msgId, postAttributeMsg)); + } else if (deviceSessionCtx.isDeviceTelemetryTopic(topicName)) { + TransportProtos.PostTelemetryMsg postTelemetryMsg = payloadAdaptor.convertToPostTelemetry(deviceSessionCtx, mqttMsg); + transportService.process(deviceSessionCtx.getSessionInfo(), postTelemetryMsg, getPubAckCallback(ctx, msgId, postTelemetryMsg)); } else if (topicName.startsWith(MqttTopics.DEVICE_ATTRIBUTES_REQUEST_TOPIC_PREFIX)) { TransportProtos.GetAttributeRequestMsg getAttributeMsg = payloadAdaptor.convertToGetAttributes(deviceSessionCtx, mqttMsg); transportService.process(deviceSessionCtx.getSessionInfo(), getAttributeMsg, getPubAckCallback(ctx, msgId, getAttributeMsg)); @@ -230,6 +298,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement transportService.process(deviceSessionCtx.getSessionInfo(), claimDeviceMsg, getPubAckCallback(ctx, msgId, claimDeviceMsg)); } else { transportService.reportActivity(deviceSessionCtx.getSessionInfo()); + ack(ctx, msgId); } } catch (AdaptorException e) { log.warn("[{}] Failed to process publish msg [{}][{}]", sessionId, topicName, msgId, e); @@ -238,15 +307,18 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } } + private void ack(ChannelHandlerContext ctx, int msgId) { + if (msgId > 0) { + ctx.writeAndFlush(createMqttPubAckMsg(msgId)); + } + } private TransportServiceCallback getPubAckCallback(final ChannelHandlerContext ctx, final int msgId, final T msg) { return new TransportServiceCallback() { @Override public void onSuccess(Void dummy) { log.trace("[{}] Published msg: {}", sessionId, msg); - if (msgId > 0) { - ctx.writeAndFlush(createMqttPubAckMsg(msgId)); - } + ack(ctx, msgId); } @Override @@ -257,6 +329,40 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement }; } + private class DeviceProvisionCallback implements TransportServiceCallback { + private final ChannelHandlerContext ctx; + private final int msgId; + private final TransportProtos.ProvisionDeviceRequestMsg msg; + + DeviceProvisionCallback(ChannelHandlerContext ctx, int msgId, TransportProtos.ProvisionDeviceRequestMsg msg) { + this.ctx = ctx; + this.msgId = msgId; + this.msg = msg; + } + + @Override + public void onSuccess(TransportProtos.ProvisionDeviceResponseMsg provisionResponseMsg) { + log.trace("[{}] Published msg: {}", sessionId, msg); + ack(ctx, msgId); + try { + if (deviceSessionCtx.getProvisionPayloadType().equals(TransportPayloadType.JSON)) { + deviceSessionCtx.getContext().getJsonMqttAdaptor().convertToPublish(deviceSessionCtx, provisionResponseMsg).ifPresent(deviceSessionCtx.getChannel()::writeAndFlush); + } else { + deviceSessionCtx.getContext().getProtoMqttAdaptor().convertToPublish(deviceSessionCtx, provisionResponseMsg).ifPresent(deviceSessionCtx.getChannel()::writeAndFlush); + } + scheduler.schedule(() -> processDisconnect(ctx), 60, TimeUnit.SECONDS); + } catch (Exception e) { + log.trace("[{}] Failed to convert device attributes response to MQTT msg", sessionId, e); + } + } + + @Override + public void onError(Throwable e) { + log.trace("[{}] Failed to publish msg: {}", sessionId, msg, e); + processDisconnect(ctx); + } + } + private void processSubscribe(ChannelHandlerContext ctx, MqttSubscribeMessage mqttMsg) { if (!checkConnected(ctx, mqttMsg)) { return; @@ -286,6 +392,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement case MqttTopics.GATEWAY_ATTRIBUTES_TOPIC: case MqttTopics.GATEWAY_RPC_TOPIC: case MqttTopics.GATEWAY_ATTRIBUTES_RESPONSE_TOPIC: + case MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC: registerSubQoS(topic, grantedQoSList, reqQoS); break; default: @@ -351,11 +458,18 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private void processConnect(ChannelHandlerContext ctx, MqttConnectMessage msg) { log.info("[{}] Processing connect msg for client: {}!", sessionId, msg.payload().clientIdentifier()); - X509Certificate cert; - if (sslHandler != null && (cert = getX509Certificate()) != null) { - processX509CertConnect(ctx, cert); + String userName = msg.payload().userName(); + String clientId = msg.payload().clientIdentifier(); + if (DataConstants.PROVISION.equals(userName) || DataConstants.PROVISION.equals(clientId)) { + deviceSessionCtx.setProvisionOnly(true); + ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED)); } else { - processAuthTokenConnect(ctx, msg); + X509Certificate cert; + if (sslHandler != null && (cert = getX509Certificate()) != null) { + processX509CertConnect(ctx, cert); + } else { + processAuthTokenConnect(ctx, msg); + } } } @@ -363,8 +477,10 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement String userName = msg.payload().userName(); log.info("[{}] Processing connect msg for client with user name: {}!", sessionId, userName); TransportProtos.ValidateBasicMqttCredRequestMsg.Builder request = TransportProtos.ValidateBasicMqttCredRequestMsg.newBuilder() - .setClientId(msg.payload().clientIdentifier()) - .setUserName(userName); + .setClientId(msg.payload().clientIdentifier()); + if (userName != null) { + request.setUserName(userName); + } String password = msg.payload().password(); if (password != null) { request.setPassword(password); @@ -387,7 +503,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private void processX509CertConnect(ChannelHandlerContext ctx, X509Certificate cert) { try { - if(!context.isSkipValidityCheckForClientCert()){ + if (!context.isSkipValidityCheckForClientCert()) { cert.checkValidity(); } String strCert = SslUtil.getX509CertificateString(cert); @@ -530,7 +646,11 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void onError(Throwable e) { - log.warn("[{}] Failed to submit session event", sessionId, e); + if (e instanceof TbRateLimitsException) { + log.trace("[{}] Failed to submit session event", sessionId, e); + } else { + log.warn("[{}] Failed to submit session event", sessionId, e); + } ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE)); ctx.close(); } @@ -583,7 +703,12 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } @Override - public void onProfileUpdate(DeviceProfile deviceProfile) { - deviceSessionCtx.onProfileUpdate(deviceProfile); + public void onDeviceProfileUpdate(TransportProtos.SessionInfoProto sessionInfo, DeviceProfile deviceProfile) { + deviceSessionCtx.onDeviceProfileUpdate(sessionInfo, deviceProfile); + } + + @Override + public void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, Device device, Optional deviceProfileOpt) { + deviceSessionCtx.onDeviceUpdate(sessionInfo, device, deviceProfileOpt); } } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java index 811e29cdae..0799d29c57 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java @@ -87,6 +87,16 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor { } } + @Override + public TransportProtos.ProvisionDeviceRequestMsg convertToProvisionRequestMsg(MqttDeviceAwareSessionContext ctx, MqttPublishMessage inbound) throws AdaptorException { + String payload = validatePayload(ctx.getSessionId(), inbound.payload(), false); + try { + return JsonConverter.convertToProvisionRequestMsg(payload); + } catch (IllegalStateException | JsonSyntaxException ex) { + throw new AdaptorException(ex); + } + } + @Override public TransportProtos.GetAttributeRequestMsg convertToGetAttributes(MqttDeviceAwareSessionContext ctx, MqttPublishMessage inbound) throws AdaptorException { return processGetAttributeRequestMsg(inbound, MqttTopics.DEVICE_ATTRIBUTES_REQUEST_TOPIC_PREFIX); @@ -138,6 +148,11 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor { return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_RPC_RESPONSE_TOPIC + rpcResponse.getRequestId(), JsonConverter.toJson(rpcResponse))); } + @Override + public Optional