Merge branch 'bug/3682-fixed' of https://github.com/dharapatel1596/thingsboard into bug/3682-fixed

application/pom.xml

@@ -89,6 +89,10 @@
             <groupId>org.thingsboard.common.transport</groupId>
             <artifactId>lwm2m</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.thingsboard.common.transport</groupId>
+            <artifactId>snmp</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.thingsboard</groupId>
             <artifactId>dao</artifactId>
@@ -141,6 +145,10 @@
             <version>${project.version}</version>
             <scope>runtime</scope>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.integration</groupId>
+            <artifactId>spring-integration-redis</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-security</artifactId>

application/src/main/data/json/system/oauth2_config_templates/apple_config.json

@@ -0,0 +1,24 @@
+{
+  "providerId": "Apple",
+  "additionalInfo": null,
+  "accessTokenUri": "https://appleid.apple.com/auth/token",
+  "authorizationUri": "https://appleid.apple.com/auth/authorize?response_mode=form_post",
+  "scope": ["email","openid","name"],
+  "jwkSetUri": "https://appleid.apple.com/auth/keys",
+  "userInfoUri": null,
+  "clientAuthenticationMethod": "POST",
+  "userNameAttributeName": "email",
+  "mapperConfig": {
+    "type": "APPLE",
+    "basic": {
+      "emailAttributeKey": "email",
+      "firstNameAttributeKey": "firstName",
+      "lastNameAttributeKey": "lastName",
+      "tenantNameStrategy": "DOMAIN"
+    }
+  },
+  "comment": null,
+  "loginButtonIcon": "apple-logo",
+  "loginButtonLabel": "Apple",
+  "helpLink": "https://developer.apple.com/sign-in-with-apple/get-started/"
+}

application/src/main/data/json/system/widget_bundles/digital_gauges.json

@@ -100,7 +100,7 @@
       "alias": "lcd_bar_gauge",
       "name": "LCD bar gauge",
       "image": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAACgCAMAAAB+IdObAAAB5lBMVEVERERFRUVGRkZHR0ZHR0dISEhJSUhJSUlKSklKSkpLS0pLS0tMTEtMTExNTUxNTU1OTk1PT05RUVBSUlFTU1JUVFNVVVNWVlVXV1ZYWFZYWFdZWVhaWllbW1lbW1pcXFpdXVtdXVxeXlxfX11gYF5gYF9hYV9iYmBjY2FkZGFkZGJlZWNmZmNmZmRnZ2RnZ2VoaGVpaWdra2hsbGlsbGptbWpubmtubmxwcG1xcW5zc29zc3B0dHF1dXJ2dnJ3d3N3d3R4eHR4eHV5eXV6enZ7e3d8fHh9fXl9fXp+fnp/f3uAgHyBgXyBgX2Cgn6Dg3+EhH+EhICFhYCHh4KHh4OJiYSJiYWKioWLi4aMjIeNjYiOjomPj4qQkIqQkIuRkYySkoySko2Tk42Tk46UlI+VlY+VlZCXl5GYmJKYmJOZmZOampSampWbm5WcnJadnZeenpifn5mgoJqhoZqhoZuiopyjo5ykpJ2kpJ6lpZ6mpp+mpqCnp6CoqKGpqaKpqaOqqqOrq6SsrKWtraWtraaurqevr6iwsKmxsamxsaqysqqzs6uzs6y0tKy0tK21ta21ta62tq63t6+3t7C4uLC5ubG6urK+vra+vre/v7e/v7jQ0MrQ0MvR0cz09PP39/b39/f///+daHfNAAAAAWJLR0ShKdSONgAABFpJREFUeNrt3f9TVFUYx/HnriCSmF9ADUqyFqUFTTuRX5KCyoOKxopiwBpQKrG2C0HSjZBFJRfUxNqFfZMpIv9pP8jaMlM/5IyxZ3uen/acnbkzrzn389xzf7lHlpceL+B4LTx6sixLDymAergkjymIWpSFwoAsCAVSClGIQhSiEIUo5CVArvu+7/t+8rafLZjq+SblHGSbiIhIuFlWyktFi1/bunPGMUgmUBUOh8Ph0UQ8Ho/H4/2BenY08mCHhenGX92BTEvjqvE5iT2QAThaB4eDDq2IL2dyh3PbqjJprx8Ov0ts3ZhDkAG5ODXkZ7LDixKB4Dv3xjae/W27dSkjnVIuIhXDK4mp2pqGRJVIw/yJipRLkDap7ug2suEWAFE5DcDUPa4Hom6135vzwHFpBmDv+jvZbrangZu90TnHnux3pBrgmhzLznSXTfcVVW7a9cCxLUrJdgDjTayMZzZ9wcYWZja3OQLJHPwE4Bd5C5j09mfnDwe5JSNwsMGVFakrvgGExQKNMrgyG183RlKGYf8hVyBDgfKOy03eliRMF7+enf3yNLDTpH4o7nYmI1crRby9CeCkXFz1z2i5yNGMQ2G/P373HwI0eVdfrBSiEIUoRCEKUYhCFKIQUk3GmI9TwKgxxlxyEZI+ApyJGGMi7UBDmzHN9oaDkL5XgOY6oNYCRVcgIS5Cep5DaixQdBkmFLKWNVMLtAwC/a1A3SzMv+8ihCQwvfpX+p6DkOk3gBN9QO+nQDAJ6f3j/z3Ef/F6doErAcDuAeqbgLJemBKFrCFkIADYWuBAE1DWB7e9cQczkvkMuBAJhUKR80BTSyj0QceEq3utTNha2zYPTFhr7VXd/a5hRhSiEH1DVIhmRCHatRSiEA27QjQjCtGupRCFaNgVsgYQefFSiEI07ArRjChEu5aGXSEadoVoRhSiXUvDrhANu0I0IwrRrqVhV4iG/V9BuowxDZPuQ+LhY8aca3Q/IzGZh/bKAoGEFZI/GYmdBG4edB/y7AtcSfchw0eARI37GRmW+9BRoZD8gXiz0FkAkNEOEwq1WvchXLHW2lu6jVeIQhSiEIUoRCEK+btK2xGgM0K0FfjqHMN2ACBiE3R3MW5t68A8TNqfv7bWWmt/siPA55F8g8zKeaBmH7YMOFRJu+zMwGyJDBCqpV/q60uDs8Rk9IIxXpUxyeo34Vvpz39IiTcCkdIs5DpTG5uIySggbUBMrrJndyb/IZv3fQg1R/6C8NGWXAi1we9kiPyDiIhIDmTD5dLUDS+aAzkrqVzINa+8jjyEHPd9f1duRtKv9pzanciBnFy3akV4T/x8hKzcWm2BDBwI0i5zzbUVnWM5kLeDqyFnJJPHkO+9U7ej61tpl7kfZf3955BLQ0e9QZcgdG2RosY07TJH9SGeQ6S4ZhAHILl19+V9vFu3KApRiEIUohCF/K8gBXJA8O/yqDAgi/KkIA7R/uOpLC8tun+s+eLT5T8Bm8H0V8ljg20AAAAASUVORK5CYII=",
-      "description": "Preconfigured gauge to display any value reading as an bar. Allows to configure value range, gradient colors and other settings.",
+      "description": "Preconfigured gauge to display any value reading as a bar. Allows to configure value range, gradient colors and other settings.",
       "descriptor": {
         "type": "latest",
         "sizeX": 2,

application/src/main/data/json/system/widget_bundles/entity_admin_widgets.json

@@ -10,7 +10,7 @@
       "alias": "device_admin_table",
       "name": "Device admin table",
       "image": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAACgCAAAAABslHx1AAAAAmJLR0QA/4ePzL8AAAmQSURBVHja7d39V1NHGsBx/7Lgafd47FH2BhAEDIFoyIL1pb5gC4upcqwW0aJUDNpSGmFXAxVQV+silUUsqQVsFvAFRF6EKEFISSAvJDf3uz8ELOtaCZCzKp35AThzZ+7cz5lnbubkCbmrCI49GXzHy5OxAKuCwy6Zd7zIruHgqjEXK6C4xlY9kVcCRH6yapAVUQYFREAEREAEREDeGsiMbWpR7bsd4d9BmyvKkOajR4+e7VBe0bC8NoKzjUrdL9WUXXlde03d7E5JalvENdvtC0Mqky9aipIPzfxvw5obS4Kcb3ppzgxty4aYTBFAMgBHWnl4I/aK9sG5Tdr8i3tRNwcJACH5Fd3wSy3z+mjqwkdmIXKUIdQm+cC2Q9JVK+60OxDKrqPgDHhLEuNyHoFSm6beYQv3CFVqpM03gP6d0haL1M2Vj6rTpT3204lx+U7Ir8CfemW3pGkMz2uqlLQXrFuljaUB0Hz5cXzKxVnIqDEu9YQ/qpBe6QH98RbHrQ2tFByDh+pn5JbAoexux8k0F9+l3nF8lRJenxc0tsk6dR/BzE96H+VL3dRK5SO29KSSYVt6Kewpwydl3Bn5coMLYGpIuvKcXnXdZGeKBTSpt0fOS7/gktrw6j+339OXRxUyIVkpygNOGmlOnqFiH+SWMCLdheC29mBqHYTSw8v43kNQkq9wW+2AEamb2lQFvt4ow5nds5BGcEhdv4XW01ag6FPQXADyDuOS2ria4ocbScrr17nJZDLt328ymUz2CCDD0l0MO0pLS/ca8CdbybwGuSX8KIVvrk+kQ6Wlpelfhbt0nS4wJtRi0c0u9to04LweOJc9C7n923IOrxFH5WGjLm92sX+7HZfUxgltaWnpIWkimhCr9IwtH5vNZrMFTh5/HO+C3BJapXAE90snzGaz+UcAvt948a4tpZaqrMghT1JK7tgOzEEqs3BJbRzLNJvNZrMriqGlHNgN+4/OVv07peIQkFvCgPQYaBzxq+fdU3eYgbRaGjd4IoZUbgdO5oGmFjhsxCW1cX5LKLq3X1/P0cReuKFuJXS+HkKbE1rCkNCHB2doVj/hcKadqc97AdjzV3+wRqpmOvW0HDyzICQUXw+WFDu21H2gyXZgi2vEJbUxGP9tCOsXSpQgkiSpDwwAijkuXWN4DJhTZsIQhrITdUmN4MpX6xMLvOFdhmZD4sHMUmhJTNz4xYIQiuI+YmqXOjnzYAZoDqdskg6Hwkdvpabokm9FaUamRkcdgdm/PQ8ehQD8EwATLiDU2+0B4Hnn6FwX770RxekEXJ1jyugM0w5gagyYeg4TLkKjfsI/AEIPBiDU1xvwjCo4fFOdA8wdDfTd80driyJ2vwIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiICsOoti94LMv8VwTdrvdbve9DRBZtQ9aVUv8lOP+dWti1q2zvh0Q1S1aVTKBniGYdD7vkX33fwXsDwKRnO37D2DCiWL3jXke2QFv1/M3BNmZ4GlVyfbkTPVJTiVo123Xpq59SllsmnY6Qsi3qTyIGdUlad+rZkC9+YPrbwbSrD/RqpLv/YObf1JOZSntql4l4dJgzJCSWRMhxB4zUL4V3RkurQ19XMgP8W8GcrvnvXKVPJmfnqqST+XQq5okw/JDTHLymuIIIWSd21KLzsKAanxDbHKCyvtmIJx6XyWXfRjqmAe5u/qZ2+mLFPJd8monOgttMTMGk9vtVN4QxJuokqvXfZ2hmnwBkQ3Z5oy2SCHO1btBt6Ei7VMa1pwp3PsmQitUNQL3qhT5ytnOqomOJpxVfq4+wFd7OpJ0K/3VANqroDN9Y/FD51ffOd/VV3Znxdop0Fne+S3K45w24HSr2GsJiIAIiIAIiIAIiIAIiIAIyIKQoRVRRGgJiIAIiIAIiIAIiID8YSG2c6/4r6uh4uLi4qF3CmLLMhS9JOkxvCg9C0OUQJSuZHnfS/R8PM9gOPdS3cUX5flrIZdSwKLN3OcH2N+mrAd7Rr81Qa/LebrAuO5Yvebg+Pya5uPLcXRkN4/nZdleqrUWzxbra0PLkRHL8KYgB64C5FiVGJwaKy25UJO7wMCu9XBNJ4N3fiJXdgJ4/It3ZBn+0jz+soPLc5F1+bWQT6yxTPZB4fU5iDfzKrTkQm96BBB2dFChzzzri/dz1tJQSEdajn6MM4Yt5sU7DIat4ywNcuOYJxbgsd43B9mjV6Alu71l298jgRy/1JMphzYPH7mFdryhUEke5NrZzq2KnOZYZFwZDIasn1ka5FfdtCcWmNA9ZhaiMu2wQEtK2d4CIoF8du1iwq5d6tsdBQO7aSh0qQH+lrhr15/bo+KIDFKXpt+8OgtPVjt0/wI7O5QYJhK6aMnFFT8WASSQNHj9iNvtDoa0Z6/TUCivD+F31h93u91yVByRrhE8sQR3lvX0DLSnP/4pzqXEwN2kyZZcqDqyEGRtj3XvSZyJbcPFbsrWe2goxFg1dqR+LLFj6LgnKo6IITOncBYVFRVV0Gg80otyDLjW+KgO/CcW+FSJt6iopBUYOJrfBMM1cP863nLjZegrzL+9yHWe3cGyIG9DOfb78/HOQX5vPqBxDtL49kNqiot/14HvXPiFvdIntvECIiACIiACIiACIiACsmiIyLOL0BIQAREQAREQAREQAXmnISLPfmcahupvht8fnrx63QNg/yWyYYNNTa3//QXKo13LcowvPc+u7ee6tvqMdgpwbqquzAiCvHlnZOO61lSdSrk5v+bhteU4OrYuPc+u7Q/EPoPj1UBXPWT1QVV+pJD1MB43iedSvdt7Awa6Rmwot853w1T9Jc+iHdnLyLNr+3sygEA4tkL3N3kY2tq5CAif/hDQX6jRyRoHBS0NhZiMNzNbfbqaC4bQoh3LyLNr++9uD0/g5SbIi/+G0M7ersVATtT9K9/t3ttlrg5sDDYUyut9DDT/s8Dt3ta7eMeS8+xo+4e0gNNhvdwEyPr7dduazmnbI4fkNddojEZj99Ot1uM0FLoSAKq0RqOxNyqOiCFy4iB8Vg9YayDv5+aqqi+Sv48YMqCetuaAB3bs7aShkDgXfT825YMnOo6IIfykKTu0fQZwpplOZAeAiEPr/aL9qR2Ecg6WbXZzeaNCQyFXsiv0D+Wdn5Xp/6959i4v/GrtDq9L388dMsB0T2TDy+3tfTIQ6mr1gLcfJgZhxDoJcmerd5EOkWd/uyAizy628QIiIAIiIALyB4esmAcEr4xHNk+OrQqsjIdoy6tWxmPNZf4DJqTD+Gup8cgAAAAASUVORK5CYII=",
-      "description": "Customized entity table widget with preconfigured actions to create update and delete devices.",
+      "description": "Customized entity table widget with preconfigured actions to create, update and delete devices.",
       "descriptor": {
         "type": "latest",
         "sizeX": 7.5,
@@ -28,7 +28,7 @@
       "alias": "asset_admin_table",
       "name": "Asset admin table",
       "image": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAACgCAAAAABslHx1AAAAAmJLR0QA/4ePzL8AAAm6SURBVHja7d3pV1PXGsBh/7LobW+txes9YYhiSsAoRKKtA63YWsHSWhwQcQATB4oo9QKptnWgqFhFMbEFLBVEoggyRMIUEwImJif53Q8BynVd5YDpUuneH5KwszmbZ+Xd+7xrveSceQQHuh+95a17IMC8YI9H5i1vsqcnOG/AwxxonoF53fJcgMjd8x4xJ9ojAREQAREQARGQNwDS2Tmj4f13J+Zv/8sgNftCszjON7nPdbRk+18yvHzV+Iv87JnP5XAogYRSpdsvPUz2aUWQ9oLA/3ac3BktiMmkBNKUkJM3/jL43HMku1x7fHJsMDyOD01AApMP4y08mZIWZUY6ghOQ4BSIHH1I3s6GBC8QKl6uXlMPoZLlaqMNaFonrbTQoFVrtJGgaf1E0uwcgdBhjSZvSy4B7amM2OXVDQYp5RrUaYMc2XFME/fVGIBXmxCrbeLJ7gT1xjYo15fo1FuHI5BwZVLshuYoQ7yaW6GUc8BpXcvg0YR+zmrvDJXE9dIeV9lfG3/L71xd5AwDDC89Otxu2AdnNDW959S5BCTj3Z6C2PSmnn0aL9ekIAfVR3vrNRUAIWfeRqefzzO7BnelhCiXCrub0r6KQP6jq+8/pB2NLuRCUpDijUD+p2Gene7lwIYw8ukedm8FCnL+DC1XnR/KDKAvBnJyCUg10C9Vg0O6G4Gsm7J6ijKBm33QJnVTrguBTd1PfjbPEs+DrL2oaJ2bTCZTVpbJZDI5Xg7JMMND6SG0JRnLOoEHuvSTD4G09UVFRRnGKWvEU7E7e7WeMelm5M8NSNfAK9nAJTVGIJunLOeiTCB4Zd+2TKk9stiHpSbys3ko5RYVFemORxXSKekMBoP6COC5lK3O8oO3Jlez5Sn6z0pLS0sr/oS4V223Nh3Q45HqlUN2Gi81VU9Ankj15Gdjlw6WlpaW2qIaWkdW19XV1RV9+AxbL/Sqr/BrDzjjqvliYi+bgFzWhKBCD8vPANsVQZ5ITdAhtVOuB+5I3eRn45Xqor79BnTlAMPqG2ze4qNTfYutm5/SE1tHtdpG6NQ5+CQ/MrZW3UpHmg6O6HtpS5wWUpwawqsuC3tyJTvlUnV4bMuGMPnZkGPsx5P7MJqQOrUzctLbRq9xWVrs/hCOj5amxe4NES6JTUkydMIZSecBkHOk5Um71WN4MyTd6sxpIS1x2t/4IXZpfJF0g3JD5oexOnvk3Sefq1M1uf5oQkb6xzfhfgg9bB4ECHfeifSOtraHADraxnOYrja/3xmE0P0HAc8Twk4fhJz+yIPPGcYzDLhdE/lVyxgMt7hwjuIdDrU3+ybfHRifIpopikjjBURABERABERABERABERABERABERABERABERABERA5hgk7HgKPscsjzXscDgcDt+bAJFVmWBVzfK/HLMWL5y/eLHtzYCormNVyQTsXeB2DdplX6sbcNwLKDnazx/AsIuwwzcw9sABPI2UJl4DZH38mFUlOxLT1PspjE9e/HGydtFjzEt0yaMKIce13Jvv1C9NfqeSTvXKD6pfD6Q2dZ9VJd89zy//DBemhxtU98PxPz2a3xVOsyiEOOZ3Fq9Bf5ifFoU27+JK3OuB1NnfKVbJ7q0rtCq5cBP3VW5WVFyZn5i4sEAhhPQTq86gr6BTNZSwJDFe9fT1QCh8VyWb14Yap0BuL+gbcfmUQr5PXOBCX0H9/GcG08iIK/yaIE81Krly8bEVKvckRDYYS1fUK4W4FmSAPqFEt41LCw/v+vR1hFaorBfuloXlc0fulA03XsVV5ufCPXxnDl1XcrSOSoDkC6A3fVvhhztHv3e9rWd2V8kiL+gr3voU5eGmeuCQVeRaAiIgAiIgAiIgAiIgAiIgAjItpGtONBFaAiIgAiIgAiIgAiIgf1tI04n/8x2yroKCgoKutwrSlG7Ie05iN0w2+0wgoQCAPPPKuwwXALgYZLZXJxoc2mIwnHiu7/RkG3wx5McckNP+OLUsNWX7GMCJlNStQU6sSN2h4LIDI0tSV6ZPXAPh0EXeA0A9Wps/O0ejsXZoS3rTc722gvFme0loBZbZqf6UkmOEdh8FHuhlMn/pSpbZ0Dj9xJ5/gTX9z5/fAzyoIwX60SAQmEmFtzHdsLp26HkHZyci6+zL1sjlTfLydkqOwc9fAa4OyKkFyFAIaTSwt4qB5ey9wHuMGtdsWDh6aRe6L9fF27mc8NH6PTNwGAxrhpgdJJz2zQ4o+bqh5sPxT+5eehDMxh0Kisyef2xct6RtKuTkATzvj17ahe5XKg/JUh+lexTHlcFgSP+NWUJofHcASozm1O8oS10LzpQeYKAx5YECyOIR9xV9eApkmxXUo5d2oeujOu+xDn7e84oOpRCXBJQcw64NArhT70BfA5grlYUW74/uOz8J2VkzFeKOVwx5sWOmEHIsgN943G7v6ltm717VqgCyyN5avBrLth7zOMRq6KxaMAlhraVj455XdCiFjB0CrDfAcRhw5uXl5ZXRvP1LJdXmp3l5e06N4DNlXTjG5WYKoCrLctTfWs23blouMrxn+4F9Ste58QXbi0LIX9rO9PkyLykYt+fFn8ebAfn9izVlYWUQ4wu3+5oJSM2bn2tZCgpefNrynYic2E/6RBovIAIiIAIiIAIiIAIiIDOGiDq7CC0BERABERABERABEZC3GiLq7DfdIF9n6OrVOieEb/1oB5CvKazGtp+96o+iY2jWdXZic8Abg01bVqytZtfX55OvAeWqx4omtqRaDiV7o+ZoXDPrOjuxmht4Y7BtgpaVwVSZmm/AYUx8zLOqyq5pJh6RPGCqocp+EWdllQ/H71Dj62ir/sE9G4fxFerssdYE7zjEZgQoshDOuJf0mOyD1ZppLgbR/HHkOSazujfxXGmaXLMTEocs8T+a9PJsHK9QZ48dPLrTG4NNnf2ZtgW4vUbm3EGSHmOsCnRM8/3h65/jNJttxIxReAqyro9DTGBsnpVj9nX22MFA8tUYbEb7D+uBjhQnQ1LV1fgzo/37k/c+e/ncf6Qz2rDbTEyAnFowVY5DzJBzPVoOxRBa4mKwbSKcbqVf3wm9ZWVlS0zDFuQtV18++bN/d8J3ZmICHP8WMqzXc5DjhiwHIeV+tBzKIRRG1sjvybI+w2w2y0DSY3JzDmt7p5neurRwv+Y3YgJ49IVfb5JdcabNC4cs8aatmVFzKIT8EQD/bdz3gaaxxoaGhoYQ0OwnbL85/dbjsd4agdsh8Dc0hWDI2tcSsJhbG+WZO968OrvFPNPfeEPr7B13ZwERdXaRxguIgAiIgAjI3xwyZ24QPDdu2ewemBeYGzfRlufNjduay/wXtgu0d9kLWo8AAAAASUVORK5CYII=",
-      "description": "Customized entity table widget with preconfigured actions to create update and delete assets.",
+      "description": "Customized entity table widget with preconfigured actions to create, update and delete assets.",
       "descriptor": {
         "type": "latest",
         "sizeX": 7.5,

application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql

@@ -18,17 +18,18 @@ CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar
     LANGUAGE plpgsql AS
 $$
 DECLARE
-    max_tenant_ttl            bigint;
-    max_customer_ttl          bigint;
-    max_ttl                   bigint;
-    date                      timestamp;
-    partition_by_max_ttl_date varchar;
-    partition_month           varchar;
-    partition_day             varchar;
-    partition_year            varchar;
-    partition                 varchar;
-    partition_to_delete       varchar;
-
+    max_tenant_ttl             bigint;
+    max_customer_ttl           bigint;
+    max_ttl                    bigint;
+    date                       timestamp;
+    partition_by_max_ttl_date  varchar;
+    partition_by_max_ttl_month varchar;
+    partition_by_max_ttl_day   varchar;
+    partition_by_max_ttl_year  varchar;
+    partition                  varchar;
+    partition_year             integer;
+    partition_month            integer;
+    partition_day              integer;
 
 BEGIN
     SELECT max(attribute_kv.long_v)
@@ -43,55 +44,140 @@ BEGIN
     into max_customer_ttl;
     max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl);
     if max_ttl IS NOT NULL AND max_ttl > 0 THEN
-        date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000));
+        date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - max_ttl);
         partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date);
+        RAISE NOTICE 'Date by max ttl: %', date;
         RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date;
         IF partition_by_max_ttl_date IS NOT NULL THEN
             CASE
                 WHEN partition_type = 'DAYS' THEN
-                    partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
-                    partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
-                    partition_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5);
+                    partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
+                    partition_by_max_ttl_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
+                    partition_by_max_ttl_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5);
                 WHEN partition_type = 'MONTHS' THEN
-                    partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
-                    partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
+                    partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
+                    partition_by_max_ttl_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
                 ELSE
-                    partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
+                    partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
                 END CASE;
-            FOR partition IN SELECT tablename
-                             FROM pg_tables
-                             WHERE schemaname = 'public'
-                               AND tablename like 'ts_kv_' || '%'
-                               AND tablename != 'ts_kv_latest'
-                               AND tablename != 'ts_kv_dictionary'
-                               AND tablename != 'ts_kv_indefinite'
-                LOOP
-                    IF partition != partition_by_max_ttl_date THEN
-                        IF partition_year IS NOT NULL THEN
-                            IF SPLIT_PART(partition, '_', 3)::integer < partition_year::integer THEN
-                                partition_to_delete := partition;
-                            ELSE
-                                IF partition_month IS NOT NULL THEN
-                                    IF SPLIT_PART(partition, '_', 4)::integer < partition_month::integer THEN
-                                        partition_to_delete := partition;
+            IF partition_by_max_ttl_year IS NULL THEN
+                RAISE NOTICE 'Failed to remove partitions by max ttl date due to partition_by_max_ttl_year is null!';
+            ELSE
+                IF partition_type = 'YEARS' THEN
+                    FOR partition IN SELECT tablename
+                                     FROM pg_tables
+                                     WHERE schemaname = 'public'
+                                       AND tablename like 'ts_kv_' || '%'
+                                       AND tablename != 'ts_kv_latest'
+                                       AND tablename != 'ts_kv_dictionary'
+                                       AND tablename != 'ts_kv_indefinite'
+                                       AND tablename != partition_by_max_ttl_date
+                        LOOP
+                            partition_year := SPLIT_PART(partition, '_', 3)::integer;
+                            IF partition_year < partition_by_max_ttl_year::integer THEN
+                                RAISE NOTICE 'Partition to delete by max ttl: %', partition;
+                                EXECUTE format('DROP TABLE IF EXISTS %I', partition);
+                                deleted := deleted + 1;
+                            END IF;
+                        END LOOP;
+                ELSE
+                    IF partition_type = 'MONTHS' THEN
+                        IF partition_by_max_ttl_month IS NULL THEN
+                            RAISE NOTICE 'Failed to remove months partitions by max ttl date due to partition_by_max_ttl_month is null!';
+                        ELSE
+                            FOR partition IN SELECT tablename
+                                             FROM pg_tables
+                                             WHERE schemaname = 'public'
+                                               AND tablename like 'ts_kv_' || '%'
+                                               AND tablename != 'ts_kv_latest'
+                                               AND tablename != 'ts_kv_dictionary'
+                                               AND tablename != 'ts_kv_indefinite'
+                                               AND tablename != partition_by_max_ttl_date
+                                LOOP
+                                    partition_year := SPLIT_PART(partition, '_', 3)::integer;
+                                    IF partition_year > partition_by_max_ttl_year::integer THEN
+                                        RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition;
+                                        CONTINUE;
                                     ELSE
-                                        IF partition_day IS NOT NULL THEN
-                                            IF SPLIT_PART(partition, '_', 5)::integer < partition_day::integer THEN
-                                                partition_to_delete := partition;
+                                        IF partition_year < partition_by_max_ttl_year::integer THEN
+                                            RAISE NOTICE 'Partition to delete by max ttl: %', partition;
+                                            EXECUTE format('DROP TABLE IF EXISTS %I', partition);
+                                            deleted := deleted + 1;
+                                        ELSE
+                                            partition_month := SPLIT_PART(partition, '_', 4)::integer;
+                                            IF partition_year = partition_by_max_ttl_year::integer THEN
+                                                IF  partition_month >= partition_by_max_ttl_month::integer THEN
+                                                    RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition;
+                                                    CONTINUE;
+                                                ELSE
+                                                    RAISE NOTICE 'Partition to delete by max ttl: %', partition;
+                                                    EXECUTE format('DROP TABLE IF EXISTS %I', partition);
+                                                    deleted := deleted + 1;
+                                                END IF;
                                             END IF;
                                         END IF;
                                     END IF;
+                                END LOOP;
+                        END IF;
+                    ELSE
+                        IF partition_type = 'DAYS' THEN
+                            IF partition_by_max_ttl_month IS NULL THEN
+                                RAISE NOTICE 'Failed to remove days partitions by max ttl date due to partition_by_max_ttl_month is null!';
+                            ELSE
+                                IF partition_by_max_ttl_day IS NULL THEN
+                                    RAISE NOTICE 'Failed to remove days partitions by max ttl date due to partition_by_max_ttl_day is null!';
+                                ELSE
+                                    FOR partition IN SELECT tablename
+                                                     FROM pg_tables
+                                                     WHERE schemaname = 'public'
+                                                       AND tablename like 'ts_kv_' || '%'
+                                                       AND tablename != 'ts_kv_latest'
+                                                       AND tablename != 'ts_kv_dictionary'
+                                                       AND tablename != 'ts_kv_indefinite'
+                                                       AND tablename != partition_by_max_ttl_date
+                                        LOOP
+                                            partition_year := SPLIT_PART(partition, '_', 3)::integer;
+                                            IF partition_year > partition_by_max_ttl_year::integer THEN
+                                                RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition;
+                                                CONTINUE;
+                                            ELSE
+                                                IF partition_year < partition_by_max_ttl_year::integer THEN
+                                                    RAISE NOTICE 'Partition to delete by max ttl: %', partition;
+                                                    EXECUTE format('DROP TABLE IF EXISTS %I', partition);
+                                                    deleted := deleted + 1;
+                                                ELSE
+                                                    partition_month := SPLIT_PART(partition, '_', 4)::integer;
+                                                    IF partition_month > partition_by_max_ttl_month::integer THEN
+                                                        RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition;
+                                                        CONTINUE;
+                                                    ELSE
+                                                        IF partition_month < partition_by_max_ttl_month::integer THEN
+                                                            RAISE NOTICE 'Partition to delete by max ttl: %', partition;
+                                                            EXECUTE format('DROP TABLE IF EXISTS %I', partition);
+                                                            deleted := deleted + 1;
+                                                        ELSE
+                                                            partition_day := SPLIT_PART(partition, '_', 5)::integer;
+                                                            IF partition_day >= partition_by_max_ttl_day::integer THEN
+                                                                RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition;
+                                                                CONTINUE;
+                                                            ELSE
+                                                                IF partition_day < partition_by_max_ttl_day::integer THEN
+                                                                    RAISE NOTICE 'Partition to delete by max ttl: %', partition;
+                                                                    EXECUTE format('DROP TABLE IF EXISTS %I', partition);
+                                                                    deleted := deleted + 1;
+                                                                END IF;
+                                                            END IF;
+                                                        END IF;
+                                                    END IF;
+                                                END IF;
+                                            END IF;
+                                        END LOOP;
                                 END IF;
                             END IF;
                         END IF;
-                        IF partition_to_delete IS NOT NULL THEN
-                            RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete;
-                            EXECUTE format('DROP TABLE IF EXISTS %I', partition_to_delete);
-                            partition_to_delete := NULL;
-                            deleted := deleted + 1;
-                        END IF;
                     END IF;
-                END LOOP;
+                END IF;
+            END IF;
         END IF;
     END IF;
 END
@@ -107,8 +193,6 @@ BEGIN
             partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM');
         WHEN partition_type = 'YEARS' THEN
             partition := 'ts_kv_' || to_char(date, 'yyyy');
-        WHEN partition_type = 'INDEFINITE' THEN
-            partition := NULL;
         ELSE
             partition := NULL;
         END CASE;

application/src/main/data/upgrade/3.2.2/schema_update.sql

@@ -31,7 +31,7 @@ CREATE TABLE IF NOT EXISTS edge (
     tenant_id uuid,
     CONSTRAINT edge_name_unq_key UNIQUE (tenant_id, name),
     CONSTRAINT edge_routing_key_unq_key UNIQUE (routing_key)
-);
+    );
 
 CREATE TABLE IF NOT EXISTS edge_event (
     id uuid NOT NULL CONSTRAINT edge_event_pkey PRIMARY KEY,
@@ -44,4 +44,170 @@ CREATE TABLE IF NOT EXISTS edge_event (
     body varchar(10000000),
     tenant_id uuid,
     ts bigint NOT NULL
+    );
+
+CREATE TABLE IF NOT EXISTS resource (
+    id uuid NOT NULL CONSTRAINT resource_pkey PRIMARY KEY,
+    created_time bigint NOT NULL,
+    tenant_id uuid NOT NULL,
+    title varchar(255) NOT NULL,
+    resource_type varchar(32) NOT NULL,
+    resource_key varchar(255) NOT NULL,
+    search_text varchar(255),
+    file_name varchar(255) NOT NULL,
+    data varchar,
+    CONSTRAINT resource_unq_key UNIQUE (tenant_id, resource_type, resource_key)
+);
+
+CREATE TABLE IF NOT EXISTS ota_package (
+    id uuid NOT NULL CONSTRAINT ota_package_pkey PRIMARY KEY,
+    created_time bigint NOT NULL,
+    tenant_id uuid NOT NULL,
+    device_profile_id uuid,
+    type varchar(32) NOT NULL,
+    title varchar(255) NOT NULL,
+    version varchar(255) NOT NULL,
+    url varchar(255),
+    file_name varchar(255),
+    content_type varchar(255),
+    checksum_algorithm varchar(32),
+    checksum varchar(1020),
+    data oid,
+    data_size bigint,
+    additional_info varchar,
+    search_text varchar(255),
+    CONSTRAINT ota_package_tenant_title_version_unq_key UNIQUE (tenant_id, title, version)
+);
+
+CREATE TABLE IF NOT EXISTS oauth2_params (
+    id uuid NOT NULL CONSTRAINT oauth2_params_pkey PRIMARY KEY,
+    enabled boolean,
+    tenant_id uuid,
+    created_time bigint NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS oauth2_registration (
+    id uuid NOT NULL CONSTRAINT oauth2_registration_pkey PRIMARY KEY,
+    oauth2_params_id uuid NOT NULL,
+    created_time bigint NOT NULL,
+    additional_info varchar,
+    client_id varchar(255),
+    client_secret varchar(2048),
+    authorization_uri varchar(255),
+    token_uri varchar(255),
+    scope varchar(255),
+    platforms varchar(255),
+    user_info_uri varchar(255),
+    user_name_attribute_name varchar(255),
+    jwk_set_uri varchar(255),
+    client_authentication_method varchar(255),
+    login_button_label varchar(255),
+    login_button_icon varchar(255),
+    allow_user_creation boolean,
+    activate_user boolean,
+    type varchar(31),
+    basic_email_attribute_key varchar(31),
+    basic_first_name_attribute_key varchar(31),
+    basic_last_name_attribute_key varchar(31),
+    basic_tenant_name_strategy varchar(31),
+    basic_tenant_name_pattern varchar(255),
+    basic_customer_name_pattern varchar(255),
+    basic_default_dashboard_name varchar(255),
+    basic_always_full_screen boolean,
+    custom_url varchar(255),
+    custom_username varchar(255),
+    custom_password varchar(255),
+    custom_send_token boolean,
+    CONSTRAINT fk_registration_oauth2_params FOREIGN KEY (oauth2_params_id) REFERENCES oauth2_params(id) ON DELETE CASCADE
 );
+
+CREATE TABLE IF NOT EXISTS oauth2_domain (
+    id uuid NOT NULL CONSTRAINT oauth2_domain_pkey PRIMARY KEY,
+    oauth2_params_id uuid NOT NULL,
+    created_time bigint NOT NULL,
+    domain_name varchar(255),
+    domain_scheme varchar(31),
+    CONSTRAINT fk_domain_oauth2_params FOREIGN KEY (oauth2_params_id) REFERENCES oauth2_params(id) ON DELETE CASCADE,
+    CONSTRAINT oauth2_domain_unq_key UNIQUE (oauth2_params_id, domain_name, domain_scheme)
+);
+
+CREATE TABLE IF NOT EXISTS oauth2_mobile (
+    id uuid NOT NULL CONSTRAINT oauth2_mobile_pkey PRIMARY KEY,
+    oauth2_params_id uuid NOT NULL,
+    created_time bigint NOT NULL,
+    pkg_name varchar(255),
+    app_secret varchar(2048),
+    CONSTRAINT fk_mobile_oauth2_params FOREIGN KEY (oauth2_params_id) REFERENCES oauth2_params(id) ON DELETE CASCADE,
+    CONSTRAINT oauth2_mobile_unq_key UNIQUE (oauth2_params_id, pkg_name)
+);
+
+ALTER TABLE dashboard
+    ADD COLUMN IF NOT EXISTS image varchar(1000000);
+
+ALTER TABLE device_profile
+    ADD COLUMN IF NOT EXISTS image varchar(1000000),
+    ADD COLUMN IF NOT EXISTS firmware_id uuid,
+    ADD COLUMN IF NOT EXISTS software_id uuid,
+    ADD COLUMN IF NOT EXISTS default_dashboard_id uuid;
+
+ALTER TABLE device
+    ADD COLUMN IF NOT EXISTS firmware_id uuid,
+    ADD COLUMN IF NOT EXISTS software_id uuid;
+
+ALTER TABLE alarm
+    ADD COLUMN IF NOT EXISTS customer_id uuid;
+
+DELETE FROM relation WHERE from_type = 'TENANT' AND relation_type_group = 'RULE_CHAIN';
+
+DO $$
+    BEGIN
+        IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_firmware_device_profile') THEN
+            ALTER TABLE device_profile
+                ADD CONSTRAINT fk_firmware_device_profile
+                    FOREIGN KEY (firmware_id) REFERENCES ota_package(id);
+        END IF;
+
+        IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_software_device_profile') THEN
+            ALTER TABLE device_profile
+                ADD CONSTRAINT fk_software_device_profile
+                    FOREIGN KEY (firmware_id) REFERENCES ota_package(id);
+        END IF;
+
+        IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_default_dashboard_device_profile') THEN
+            ALTER TABLE device_profile
+                ADD CONSTRAINT fk_default_dashboard_device_profile
+                    FOREIGN KEY (default_dashboard_id) REFERENCES dashboard(id);
+        END IF;
+
+        IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_firmware_device') THEN
+            ALTER TABLE device
+                ADD CONSTRAINT fk_firmware_device
+                    FOREIGN KEY (firmware_id) REFERENCES ota_package(id);
+        END IF;
+
+        IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_software_device') THEN
+            ALTER TABLE device
+                ADD CONSTRAINT fk_software_device
+                    FOREIGN KEY (firmware_id) REFERENCES ota_package(id);
+        END IF;
+    END;
+$$;
+
+
+ALTER TABLE api_usage_state
+    ADD COLUMN IF NOT EXISTS alarm_exec VARCHAR(32);
+UPDATE api_usage_state SET alarm_exec = 'ENABLED' WHERE alarm_exec IS NULL;
+
+CREATE TABLE IF NOT EXISTS rpc (
+    id uuid NOT NULL CONSTRAINT rpc_pkey PRIMARY KEY,
+    created_time bigint NOT NULL,
+    tenant_id uuid NOT NULL,
+    device_id uuid NOT NULL,
+    expiration_time bigint NOT NULL,
+    request varchar(10000000) NOT NULL,
+    response varchar(10000000),
+    status varchar(255) NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id);
+

application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@@ -60,9 +60,12 @@ import org.thingsboard.server.dao.edge.EdgeService;
 import org.thingsboard.server.dao.entityview.EntityViewService;
 import org.thingsboard.server.dao.event.EventService;
 import org.thingsboard.server.dao.nosql.CassandraBufferedRateExecutor;
+import org.thingsboard.server.dao.ota.OtaPackageService;
 import org.thingsboard.server.dao.relation.RelationService;
+import org.thingsboard.server.dao.resource.ResourceService;
 import org.thingsboard.server.dao.rule.RuleChainService;
 import org.thingsboard.server.dao.rule.RuleNodeStateService;
+import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
 import org.thingsboard.server.dao.tenant.TenantProfileService;
 import org.thingsboard.server.dao.tenant.TenantService;
 import org.thingsboard.server.dao.timeseries.TimeseriesService;
@@ -78,9 +81,9 @@ import org.thingsboard.server.service.executors.ExternalCallExecutorService;
 import org.thingsboard.server.service.executors.SharedEventLoopGroupService;
 import org.thingsboard.server.service.mail.MailExecutorService;
 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
-import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
 import org.thingsboard.server.service.queue.TbClusterService;
 import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
+import org.thingsboard.server.service.rpc.TbRpcService;
 import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
 import org.thingsboard.server.service.script.JsExecutorService;
 import org.thingsboard.server.service.script.JsInvokeService;
@@ -301,15 +304,33 @@ public class ActorSystemContext {
 
     @Lazy
     @Autowired(required = false)
-    @Getter private EdgeService edgeService;
+    @Getter
+    private EdgeService edgeService;
 
     @Lazy
     @Autowired(required = false)
-    @Getter private EdgeEventService edgeEventService;
+    @Getter
+    private EdgeEventService edgeEventService;
 
     @Lazy
     @Autowired(required = false)
-    @Getter private EdgeRpcService edgeRpcService;
+    @Getter
+    private EdgeRpcService edgeRpcService;
+
+    @Lazy
+    @Autowired(required = false)
+    @Getter
+    private ResourceService resourceService;
+
+    @Lazy
+    @Autowired(required = false)
+    @Getter
+    private OtaPackageService otaPackageService;
+
+    @Lazy
+    @Autowired(required = false)
+    @Getter
+    private TbRpcService tbRpcService;
 
     @Value("${actors.session.max_concurrent_sessions_per_device:1}")
     @Getter

application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java

@@ -46,7 +46,7 @@ public class DeviceActor extends ContextAwareActor {
         super.init(ctx);
         log.debug("[{}][{}] Starting device actor.", processor.tenantId, processor.deviceId);
         try {
-            processor.initSessionTimeout(ctx);
+            processor.init(ctx);
             log.debug("[{}][{}] Device actor started.", processor.tenantId, processor.deviceId);
         } catch (Exception e) {
             log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.deviceId, e);

application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java

@@ -23,10 +23,11 @@ import com.google.common.util.concurrent.MoreExecutors;
 import com.google.protobuf.InvalidProtocolBufferException;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
+import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.rule.engine.api.RpcError;
 import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg;
-import org.thingsboard.rule.engine.api.msg.DeviceEdgeUpdateMsg;
 import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg;
+import org.thingsboard.rule.engine.api.msg.DeviceEdgeUpdateMsg;
 import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg;
 import org.thingsboard.server.actors.ActorSystemContext;
 import org.thingsboard.server.actors.TbActorCtx;
@@ -38,12 +39,17 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType;
 import org.thingsboard.server.common.data.edge.EdgeEventType;
 import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.EdgeId;
+import org.thingsboard.server.common.data.id.RpcId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.kv.AttributeKey;
 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
 import org.thingsboard.server.common.data.kv.KvEntry;
+import org.thingsboard.server.common.data.page.PageData;
+import org.thingsboard.server.common.data.page.PageLink;
 import org.thingsboard.server.common.data.relation.EntityRelation;
 import org.thingsboard.server.common.data.relation.RelationTypeGroup;
+import org.thingsboard.server.common.data.rpc.Rpc;
+import org.thingsboard.server.common.data.rpc.RpcStatus;
 import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody;
 import org.thingsboard.server.common.data.security.DeviceCredentials;
 import org.thingsboard.server.common.data.security.DeviceCredentialsType;
@@ -52,8 +58,8 @@ import org.thingsboard.server.common.msg.TbMsgMetaData;
 import org.thingsboard.server.common.msg.queue.TbCallback;
 import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest;
 import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg;
-import org.thingsboard.server.gen.transport.TransportProtos;
 import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg;
+import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry;
 import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg;
@@ -68,12 +74,13 @@ import org.thingsboard.server.gen.transport.TransportProtos.SessionType;
 import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto;
+import org.thingsboard.server.gen.transport.TransportProtos.ToDevicePersistedRpcResponseMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
-import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportUpdateCredentialsProto;
+import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto;
 import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
 import org.thingsboard.server.service.rpc.FromDeviceRpcResponseActorMsg;
@@ -163,14 +170,19 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
 
     void processRpcRequest(TbActorCtx context, ToDeviceRpcRequestActorMsg msg) {
         ToDeviceRpcRequest request = msg.getMsg();
-        ToDeviceRpcRequestBody body = request.getBody();
-        ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder().setRequestId(
-                rpcSeq++).setMethodName(body.getMethod()).setParams(body.getParams()).build();
+        ToDeviceRpcRequestMsg rpcRequest = creteToDeviceRpcRequestMsg(request);
 
         long timeout = request.getExpirationTime() - System.currentTimeMillis();
+        boolean persisted = request.isPersisted();
+
         if (timeout <= 0) {
             log.debug("[{}][{}] Ignoring message due to exp time reached, {}", deviceId, request.getId(), request.getExpirationTime());
+            if (persisted) {
+                createRpc(request, RpcStatus.TIMEOUT);
+            }
             return;
+        } else if (persisted) {
+            createRpc(request, RpcStatus.QUEUED);
         }
 
         boolean sent;
@@ -187,9 +199,16 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
                     syncSessionSet.add(key);
                 }
             });
+            log.trace("46) Rpc syncSessionSet [{}] subscription after sent [{}]", syncSessionSet, rpcSubscriptions);
             syncSessionSet.forEach(rpcSubscriptions::remove);
         }
 
+        if (persisted && !(sent || request.isOneway())) {
+            ObjectNode response = JacksonUtil.newObjectNode();
+            response.put("rpcId", request.getId().toString());
+            systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), JacksonUtil.toString(response), null));
+        }
+
         if (request.isOneway() && sent) {
             log.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId());
             systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null));
@@ -203,6 +222,32 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
         }
     }
 
+    private Rpc createRpc(ToDeviceRpcRequest request, RpcStatus status) {
+        Rpc rpc = new Rpc(new RpcId(request.getId()));
+        rpc.setCreatedTime(System.currentTimeMillis());
+        rpc.setTenantId(tenantId);
+        rpc.setDeviceId(deviceId);
+        rpc.setExpirationTime(request.getExpirationTime());
+        rpc.setRequest(JacksonUtil.valueToTree(request));
+        rpc.setStatus(status);
+        systemContext.getTbRpcService().save(tenantId, rpc);
+        return systemContext.getTbRpcService().save(tenantId, rpc);
+    }
+
+    private ToDeviceRpcRequestMsg creteToDeviceRpcRequestMsg(ToDeviceRpcRequest request) {
+        ToDeviceRpcRequestBody body = request.getBody();
+        return ToDeviceRpcRequestMsg.newBuilder()
+                .setRequestId(rpcSeq++)
+                .setMethodName(body.getMethod())
+                .setParams(body.getParams())
+                .setExpirationTime(request.getExpirationTime())
+                .setRequestIdMSB(request.getId().getMostSignificantBits())
+                .setRequestIdLSB(request.getId().getLeastSignificantBits())
+                .setOneway(request.isOneway())
+                .setPersisted(request.isPersisted())
+                .build();
+    }
+
     void processRpcResponsesFromEdge(TbActorCtx context, FromDeviceRpcResponseActorMsg responseMsg) {
         log.debug("[{}] Processing rpc command response from edge session", deviceId);
         ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(responseMsg.getRequestId());
@@ -224,6 +269,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
         ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(msg.getId());
         if (requestMd != null) {
             log.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId());
+            systemContext.getTbRpcService().save(tenantId, new RpcId(requestMd.getMsg().getMsg().getId()), RpcStatus.TIMEOUT, null);
             systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(),
                     null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION));
         }
@@ -258,8 +304,17 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
                 sentOneWayIds.add(entry.getKey());
                 systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null));
             }
-            ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder().setRequestId(
-                    entry.getKey()).setMethodName(body.getMethod()).setParams(body.getParams()).build();
+            ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder()
+                    .setRequestId(entry.getKey())
+                    .setMethodName(body.getMethod())
+                    .setParams(body.getParams())
+                    .setExpirationTime(request.getExpirationTime())
+                    .setRequestIdMSB(request.getId().getMostSignificantBits())
+                    .setRequestIdLSB(request.getId().getLeastSignificantBits())
+                    .setOneway(request.isOneway())
+                    .setPersisted(request.isPersisted())
+                    .build();
+
             sendToTransport(rpcRequest, sessionId, nodeId);
         };
     }
@@ -267,31 +322,39 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
     void process(TbActorCtx context, TransportToDeviceActorMsgWrapper wrapper) {
         TransportToDeviceActorMsg msg = wrapper.getMsg();
         TbCallback callback = wrapper.getCallback();
+        var sessionInfo = msg.getSessionInfo();
+
         if (msg.hasSessionEvent()) {
-            processSessionStateMsgs(msg.getSessionInfo(), msg.getSessionEvent());
+            processSessionStateMsgs(sessionInfo, msg.getSessionEvent());
         }
         if (msg.hasSubscribeToAttributes()) {
-            processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToAttributes());
+            processSubscriptionCommands(context, sessionInfo, msg.getSubscribeToAttributes());
         }
         if (msg.hasSubscribeToRPC()) {
-            processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToRPC());
+            processSubscriptionCommands(context, sessionInfo, msg.getSubscribeToRPC());
+        }
+        if (msg.hasSendPendingRPC()) {
+            sendPendingRequests(context, getSessionId(sessionInfo), sessionInfo);
         }
         if (msg.hasGetAttributes()) {
-            handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes());
+            handleGetAttributesRequest(context, sessionInfo, msg.getGetAttributes());
         }
         if (msg.hasToDeviceRPCCallResponse()) {
-            processRpcResponses(context, msg.getSessionInfo(), msg.getToDeviceRPCCallResponse());
+            processRpcResponses(context, sessionInfo, msg.getToDeviceRPCCallResponse());
         }
         if (msg.hasSubscriptionInfo()) {
-            handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo());
+            handleSessionActivity(context, sessionInfo, msg.getSubscriptionInfo());
         }
         if (msg.hasClaimDevice()) {
-            handleClaimDeviceMsg(context, msg.getSessionInfo(), msg.getClaimDevice());
+            handleClaimDeviceMsg(context, sessionInfo, msg.getClaimDevice());
+        }
+        if (msg.hasPersistedRpcResponseMsg()) {
+            processPersistedRpcResponses(context, sessionInfo, msg.getPersistedRpcResponseMsg());
         }
         callback.onSuccess();
     }
 
-    private void handleClaimDeviceMsg(TbActorCtx context, SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg) {
+    private void handleClaimDeviceMsg(TbActorCtx context, SessionInfoProto sessionInfo, ClaimDeviceMsg msg) {
         DeviceId deviceId = new DeviceId(new UUID(msg.getDeviceIdMSB(), msg.getDeviceIdLSB()));
         systemContext.getClaimDevicesService().registerClaimingInfo(tenantId, deviceId, msg.getSecretKey(), msg.getDurationMs());
     }
@@ -306,25 +369,48 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
 
     private void handleGetAttributesRequest(TbActorCtx context, SessionInfoProto sessionInfo, GetAttributeRequestMsg request) {
         int requestId = request.getRequestId();
-        Futures.addCallback(getAttributesKvEntries(request), new FutureCallback<List<List<AttributeKvEntry>>>() {
-            @Override
-            public void onSuccess(@Nullable List<List<AttributeKvEntry>> result) {
-                GetAttributeResponseMsg responseMsg = GetAttributeResponseMsg.newBuilder()
-                        .setRequestId(requestId)
-                        .addAllClientAttributeList(toTsKvProtos(result.get(0)))
-                        .addAllSharedAttributeList(toTsKvProtos(result.get(1)))
-                        .build();
-                sendToTransport(responseMsg, sessionInfo);
-            }
+        if (request.getOnlyShared()) {
+            Futures.addCallback(findAllAttributesByScope(DataConstants.SHARED_SCOPE), new FutureCallback<>() {
+                @Override
+                public void onSuccess(@Nullable List<AttributeKvEntry> result) {
+                    GetAttributeResponseMsg responseMsg = GetAttributeResponseMsg.newBuilder()
+                            .setRequestId(requestId)
+                            .setSharedStateMsg(true)
+                            .addAllSharedAttributeList(toTsKvProtos(result))
+                            .build();
+                    sendToTransport(responseMsg, sessionInfo);
+                }
 
-            @Override
-            public void onFailure(Throwable t) {
-                GetAttributeResponseMsg responseMsg = GetAttributeResponseMsg.newBuilder()
-                        .setError(t.getMessage())
-                        .build();
-                sendToTransport(responseMsg, sessionInfo);
-            }
-        }, MoreExecutors.directExecutor());
+                @Override
+                public void onFailure(Throwable t) {
+                    GetAttributeResponseMsg responseMsg = GetAttributeResponseMsg.newBuilder()
+                            .setError(t.getMessage())
+                            .setSharedStateMsg(true)
+                            .build();
+                    sendToTransport(responseMsg, sessionInfo);
+                }
+            }, MoreExecutors.directExecutor());
+        } else {
+            Futures.addCallback(getAttributesKvEntries(request), new FutureCallback<>() {
+                @Override
+                public void onSuccess(@Nullable List<List<AttributeKvEntry>> result) {
+                    GetAttributeResponseMsg responseMsg = GetAttributeResponseMsg.newBuilder()
+                            .setRequestId(requestId)
+                            .addAllClientAttributeList(toTsKvProtos(result.get(0)))
+                            .addAllSharedAttributeList(toTsKvProtos(result.get(1)))
+                            .build();
+                    sendToTransport(responseMsg, sessionInfo);
+                }
+
+                @Override
+                public void onFailure(Throwable t) {
+                    GetAttributeResponseMsg responseMsg = GetAttributeResponseMsg.newBuilder()
+                            .setError(t.getMessage())
+                            .build();
+                    sendToTransport(responseMsg, sessionInfo);
+                }
+            }, MoreExecutors.directExecutor());
+        }
     }
 
     private ListenableFuture<List<List<AttributeKvEntry>>> getAttributesKvEntries(GetAttributeRequestMsg request) {
@@ -392,9 +478,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
             }
             if (hasNotificationData) {
                 AttributeUpdateNotificationMsg finalNotification = notification.build();
-                attributeSubscriptions.entrySet().forEach(sub -> {
-                    sendToTransport(finalNotification, sub.getKey(), sub.getValue().getNodeId());
-                });
+                attributeSubscriptions.forEach((key, value) -> sendToTransport(finalNotification, key, value.getNodeId()));
             }
         } else {
             log.debug("[{}] No registered attributes subscriptions to process!", deviceId);
@@ -409,11 +493,22 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
         if (success) {
             systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(),
                     responseMsg.getPayload(), null));
+            if (requestMd.getMsg().getMsg().isPersisted()) {
+                systemContext.getTbRpcService().save(tenantId, new RpcId(requestMd.getMsg().getMsg().getId()), RpcStatus.SUCCESSFUL, JacksonUtil.toJsonNode(responseMsg.getPayload()));
+            }
         } else {
             log.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId());
+            if (requestMd.getMsg().getMsg().isPersisted()) {
+                systemContext.getTbRpcService().save(tenantId, new RpcId(requestMd.getMsg().getMsg().getId()), RpcStatus.FAILED, JacksonUtil.toJsonNode(responseMsg.getPayload()));
+            }
         }
     }
 
+    private void processPersistedRpcResponses(TbActorCtx context, SessionInfoProto sessionInfo, ToDevicePersistedRpcResponseMsg responseMsg) {
+        UUID rpcId = new UUID(responseMsg.getRequestIdMSB(), responseMsg.getRequestIdLSB());
+        systemContext.getTbRpcService().save(tenantId, new RpcId(rpcId), RpcStatus.valueOf(responseMsg.getStatus()), null);
+    }
+
     private void processSubscriptionCommands(TbActorCtx context, SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg subscribeCmd) {
         UUID sessionId = getSessionId(sessionInfo);
         if (subscribeCmd.getUnsubscribe()) {
@@ -422,7 +517,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
         } else {
             SessionInfoMetaData sessionMD = sessions.get(sessionId);
             if (sessionMD == null) {
-                sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId()));
+                sessionMD = new SessionInfoMetaData(new SessionInfo(subscribeCmd.getSessionType(), sessionInfo.getNodeId()));
             }
             sessionMD.setSubscribedToAttributes(true);
             log.debug("[{}] Registering attributes subscription for session [{}]", deviceId, sessionId);
@@ -443,7 +538,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
         } else {
             SessionInfoMetaData sessionMD = sessions.get(sessionId);
             if (sessionMD == null) {
-                sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId()));
+                sessionMD = new SessionInfoMetaData(new SessionInfo(subscribeCmd.getSessionType(), sessionInfo.getNodeId()));
             }
             sessionMD.setSubscribedToRPC(true);
             log.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId);
@@ -464,7 +559,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
             if (sessions.size() >= systemContext.getMaxConcurrentSessionsPerDevice()) {
                 UUID sessionIdToRemove = sessions.keySet().stream().findFirst().orElse(null);
                 if (sessionIdToRemove != null) {
-                    notifyTransportAboutClosedSession(sessionIdToRemove, sessions.remove(sessionIdToRemove));
+                    notifyTransportAboutClosedSession(sessionIdToRemove, sessions.remove(sessionIdToRemove), "max concurrent sessions limit reached per device!");
                 }
             }
             sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfo.getNodeId())));
@@ -510,7 +605,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
                 notifyTransportAboutProfileUpdate(k, v, ((DeviceCredentialsUpdateNotificationMsg) msg).getDeviceCredentials());
             });
         } else {
-            sessions.forEach(this::notifyTransportAboutClosedSession);
+            sessions.forEach((sessionId, sessionMd) -> notifyTransportAboutClosedSession(sessionId, sessionMd, "device credentials updated!"));
             attributeSubscriptions.clear();
             rpcSubscriptions.clear();
             dumpSessions();
@@ -518,17 +613,21 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
         }
     }
 
-    private void notifyTransportAboutClosedSession(UUID sessionId, SessionInfoMetaData sessionMd) {
+    private void notifyTransportAboutClosedSession(UUID sessionId, SessionInfoMetaData sessionMd, String message) {
+        SessionCloseNotificationProto sessionCloseNotificationProto = SessionCloseNotificationProto
+                .newBuilder()
+                .setMessage(message).build();
         ToTransportMsg msg = ToTransportMsg.newBuilder()
                 .setSessionIdMSB(sessionId.getMostSignificantBits())
                 .setSessionIdLSB(sessionId.getLeastSignificantBits())
-                .setSessionCloseNotification(SessionCloseNotificationProto.getDefaultInstance()).build();
+                .setSessionCloseNotification(sessionCloseNotificationProto)
+                .build();
         systemContext.getTbCoreToTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg);
     }
 
     void notifyTransportAboutProfileUpdate(UUID sessionId, SessionInfoMetaData sessionMd, DeviceCredentials deviceCredentials) {
         log.info("2) LwM2Mtype: ");
-        TransportProtos.ToTransportUpdateCredentialsProto.Builder notification = TransportProtos.ToTransportUpdateCredentialsProto.newBuilder();
+        ToTransportUpdateCredentialsProto.Builder notification = ToTransportUpdateCredentialsProto.newBuilder();
         notification.addCredentialsId(deviceCredentials.getCredentialsId());
         notification.addCredentialsValue(deviceCredentials.getCredentialsValue());
         ToTransportMsg msg = ToTransportMsg.newBuilder()
@@ -603,7 +702,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
         ListenableFuture<EdgeEvent> future = systemContext.getEdgeEventService().saveAsync(edgeEvent);
         Futures.addCallback(future, new FutureCallback<EdgeEvent>() {
             @Override
-            public void onSuccess( EdgeEvent result) {
+            public void onSuccess(EdgeEvent result) {
                 systemContext.getClusterService().onEdgeEventUpdate(tenantId, edgeId);
             }
 
@@ -719,8 +818,26 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
                         .addAllSessions(sessionsList).build().toByteArray());
     }
 
-    void initSessionTimeout(TbActorCtx ctx) {
-        schedulePeriodicMsgWithDelay(ctx, SessionTimeoutCheckMsg.instance(), systemContext.getSessionInactivityTimeout(), systemContext.getSessionInactivityTimeout());
+    void init(TbActorCtx ctx) {
+        schedulePeriodicMsgWithDelay(ctx, SessionTimeoutCheckMsg.instance(), systemContext.getSessionReportTimeout(), systemContext.getSessionReportTimeout());
+        PageLink pageLink = new PageLink(1024);
+        PageData<Rpc> pageData;
+        do {
+            pageData = systemContext.getTbRpcService().findAllByDeviceIdAndStatus(tenantId, deviceId, RpcStatus.QUEUED, pageLink);
+            pageData.getData().forEach(rpc -> {
+                ToDeviceRpcRequest msg = JacksonUtil.convertValue(rpc.getRequest(), ToDeviceRpcRequest.class);
+                long timeout = rpc.getExpirationTime() - System.currentTimeMillis();
+                if (timeout <= 0) {
+                    rpc.setStatus(RpcStatus.TIMEOUT);
+                    systemContext.getTbRpcService().save(tenantId, rpc);
+                } else {
+                    registerPendingRpcRequest(ctx, new ToDeviceRpcRequestActorMsg(systemContext.getServiceId(), msg), false, creteToDeviceRpcRequestMsg(msg), timeout);
+                }
+            });
+            if (pageData.hasNext()) {
+                pageLink = pageLink.nextPageLink();
+            }
+        } while (pageData.hasNext());
     }
 
     void checkSessionsTimeout() {
@@ -730,7 +847,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
             sessions.remove(sessionId);
             rpcSubscriptions.remove(sessionId);
             attributeSubscriptions.remove(sessionId);
-            notifyTransportAboutClosedSession(sessionId, sessionMD);
+            notifyTransportAboutClosedSession(sessionId, sessionMD, "session timeout!");
         });
         if (!sessionsToRemove.isEmpty()) {
             dumpSessions();

application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.TenantProfile;
 import org.thingsboard.server.common.data.alarm.Alarm;
 import org.thingsboard.server.common.data.asset.Asset;
+import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.EdgeId;
 import org.thingsboard.server.common.data.id.EntityId;
@@ -68,7 +69,9 @@ import org.thingsboard.server.dao.edge.EdgeService;
 import org.thingsboard.server.dao.entityview.EntityViewService;
 import org.thingsboard.server.dao.nosql.CassandraStatementTask;
 import org.thingsboard.server.dao.nosql.TbResultSetFuture;
+import org.thingsboard.server.dao.ota.OtaPackageService;
 import org.thingsboard.server.dao.relation.RelationService;
+import org.thingsboard.server.dao.resource.ResourceService;
 import org.thingsboard.server.dao.rule.RuleChainService;
 import org.thingsboard.server.dao.tenant.TenantService;
 import org.thingsboard.server.dao.timeseries.TimeseriesService;
@@ -269,7 +272,12 @@ class DefaultTbContext implements TbContext {
 
     @Override
     public TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) {
-        return TbMsg.newMsg(queueName, type, originator, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId());
+        return newMsg(queueName, type, originator, null, metaData, data);
+    }
+
+    @Override
+    public TbMsg newMsg(String queueName, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data) {
+        return TbMsg.newMsg(queueName, type, originator, customerId, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId());
     }
 
     @Override
@@ -480,6 +488,16 @@ class DefaultTbContext implements TbContext {
         return mainCtx.getEntityViewService();
     }
 
+    @Override
+    public ResourceService getResourceService() {
+        return mainCtx.getResourceService();
+    }
+
+    @Override
+    public OtaPackageService getOtaPackageService() {
+        return mainCtx.getOtaPackageService();
+    }
+
     @Override
     public RuleEngineDeviceProfileCache getDeviceProfileCache() {
         return mainCtx.getDeviceProfileCache();

application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java

@@ -106,7 +106,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
         int ruleNodeCount = tbMsg.getAndIncrementRuleNodeCounter();
         int maxRuleNodeExecutionsPerMessage = getTenantProfileConfiguration().getMaxRuleNodeExecsPerMessage();
         if (maxRuleNodeExecutionsPerMessage == 0 || ruleNodeCount < maxRuleNodeExecutionsPerMessage) {
-            apiUsageClient.report(tenantId, ApiUsageRecordKey.RE_EXEC_COUNT);
+            apiUsageClient.report(tenantId, tbMsg.getCustomerId(), ApiUsageRecordKey.RE_EXEC_COUNT);
             if (ruleNode.isDebugMode()) {
                 systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), "Self");
             }
@@ -127,7 +127,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
         int ruleNodeCount = tbMsg.getAndIncrementRuleNodeCounter();
         int maxRuleNodeExecutionsPerMessage = getTenantProfileConfiguration().getMaxRuleNodeExecsPerMessage();
         if (maxRuleNodeExecutionsPerMessage == 0 || ruleNodeCount < maxRuleNodeExecutionsPerMessage) {
-            apiUsageClient.report(tenantId, ApiUsageRecordKey.RE_EXEC_COUNT);
+            apiUsageClient.report(tenantId, tbMsg.getCustomerId(), ApiUsageRecordKey.RE_EXEC_COUNT);
             if (ruleNode.isDebugMode()) {
                 systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), msg.getFromRelationType());
             }

application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java

@@ -22,10 +22,10 @@ import org.springframework.boot.context.event.ApplicationReadyEvent;
 import org.springframework.context.event.EventListener;
 import org.springframework.core.annotation.Order;
 import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.ThingsBoardExecutors;
 import org.thingsboard.common.util.ThingsBoardThreadFactory;
 import org.thingsboard.server.actors.ActorSystemContext;
 import org.thingsboard.server.actors.DefaultTbActorSystem;
-import org.thingsboard.server.actors.TbActorId;
 import org.thingsboard.server.actors.TbActorRef;
 import org.thingsboard.server.actors.TbActorSystem;
 import org.thingsboard.server.actors.TbActorSystemSettings;
@@ -33,14 +33,13 @@ import org.thingsboard.server.actors.app.AppActor;
 import org.thingsboard.server.actors.app.AppInitMsg;
 import org.thingsboard.server.actors.stats.StatsActor;
 import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
-import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
 import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
+import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
 
 @Service
 @Slf4j
@@ -110,7 +109,7 @@ public class DefaultActorService extends TbApplicationEventListener<PartitionCha
         if (poolSize == 1) {
             return Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(dispatcherName));
         } else {
-            return Executors.newWorkStealingPool(poolSize);
+            return ThingsBoardExecutors.newWorkStealingPool(poolSize, dispatcherName);
         }
     }
 

application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java

@@ -37,6 +37,9 @@ import org.springframework.util.StringUtils;
 import org.springframework.web.util.UriComponents;
 import org.springframework.web.util.UriComponentsBuilder;
 import org.thingsboard.server.dao.oauth2.OAuth2Configuration;
+import org.thingsboard.server.dao.oauth2.OAuth2Service;
+import org.thingsboard.server.service.security.auth.oauth2.TbOAuth2ParameterNames;
+import org.thingsboard.server.service.security.model.token.OAuth2AppTokenFactory;
 import org.thingsboard.server.utils.MiscUtils;
 
 import javax.servlet.http.HttpServletRequest;
@@ -46,12 +49,13 @@ import java.security.NoSuchAlgorithmException;
 import java.util.Base64;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.UUID;
 
 @Service
 @Slf4j
 public class CustomOAuth2AuthorizationRequestResolver implements OAuth2AuthorizationRequestResolver {
-    public static final String DEFAULT_AUTHORIZATION_REQUEST_BASE_URI = "/oauth2/authorization";
-    public static final String DEFAULT_LOGIN_PROCESSING_URI = "/login/oauth2/code/";
+    private static final String DEFAULT_AUTHORIZATION_REQUEST_BASE_URI = "/oauth2/authorization";
+    private static final String DEFAULT_LOGIN_PROCESSING_URI = "/login/oauth2/code/";
     private static final String REGISTRATION_ID_URI_VARIABLE_NAME = "registrationId";
     private static final char PATH_DELIMITER = '/';
 
@@ -63,6 +67,12 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza
     @Autowired
     private ClientRegistrationRepository clientRegistrationRepository;
 
+    @Autowired
+    private OAuth2Service oAuth2Service;
+
+    @Autowired
+    private OAuth2AppTokenFactory oAuth2AppTokenFactory;
+
     @Autowired(required = false)
     private OAuth2Configuration oauth2Configuration;
 
@@ -71,7 +81,9 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza
     public OAuth2AuthorizationRequest resolve(HttpServletRequest request) {
         String registrationId = this.resolveRegistrationId(request);
         String redirectUriAction = getAction(request, "login");
-        return resolve(request, registrationId, redirectUriAction);
+        String appPackage = getAppPackage(request);
+        String appToken = getAppToken(request);
+        return resolve(request, registrationId, redirectUriAction, appPackage, appToken);
     }
 
     @Override
@@ -80,7 +92,9 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza
             return null;
         }
         String redirectUriAction = getAction(request, "authorize");
-        return resolve(request, registrationId, redirectUriAction);
+        String appPackage = getAppPackage(request);
+        String appToken = getAppToken(request);
+        return resolve(request, registrationId, redirectUriAction, appPackage, appToken);
     }
 
     private String getAction(HttpServletRequest request, String defaultAction) {
@@ -91,8 +105,16 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza
         return action;
     }
 
+    private String getAppPackage(HttpServletRequest request) {
+        return request.getParameter("pkg");
+    }
+
+    private String getAppToken(HttpServletRequest request) {
+        return request.getParameter("appToken");
+    }
+
     @SuppressWarnings("deprecation")
-    private OAuth2AuthorizationRequest resolve(HttpServletRequest request, String registrationId, String redirectUriAction) {
+    private OAuth2AuthorizationRequest resolve(HttpServletRequest request, String registrationId, String redirectUriAction, String appPackage, String appToken) {
         if (registrationId == null) {
             return null;
         }
@@ -104,6 +126,18 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza
 
         Map<String, Object> attributes = new HashMap<>();
         attributes.put(OAuth2ParameterNames.REGISTRATION_ID, clientRegistration.getRegistrationId());
+        if (!StringUtils.isEmpty(appPackage)) {
+            if (StringUtils.isEmpty(appToken)) {
+                throw new IllegalArgumentException("Invalid application token.");
+            } else {
+                String appSecret = this.oAuth2Service.findAppSecret(UUID.fromString(registrationId), appPackage);
+                if (StringUtils.isEmpty(appSecret)) {
+                    throw new IllegalArgumentException("Invalid package: " + appPackage + ". No application secret found for Client Registration with given application package.");
+                }
+                String callbackUrlScheme = this.oAuth2AppTokenFactory.validateTokenAndGetCallbackUrlScheme(appPackage, appToken, appSecret);
+                attributes.put(TbOAuth2ParameterNames.CALLBACK_URL_SCHEME, callbackUrlScheme);
+            }
+        }
 
         OAuth2AuthorizationRequest.Builder builder;
         if (AuthorizationGrantType.AUTHORIZATION_CODE.equals(clientRegistration.getAuthorizationGrantType())) {

application/src/main/java/org/thingsboard/server/controller/AlarmController.java

@@ -46,8 +46,6 @@ import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.security.permission.Operation;
 import org.thingsboard.server.service.security.permission.Resource;
 
-import java.util.UUID;
-
 @RestController
 @TbCoreComponent
 @RequestMapping("/api")
@@ -112,8 +110,11 @@ public class AlarmController extends BaseController {
         checkParameter(ALARM_ID, strAlarmId);
         try {
             AlarmId alarmId = new AlarmId(toUUID(strAlarmId));
-            checkAlarmId(alarmId, Operation.WRITE);
+            Alarm alarm = checkAlarmId(alarmId, Operation.WRITE);
 
+            logEntityAction(alarm.getOriginator(), alarm,
+                    getCurrentUser().getCustomerId(),
+                    ActionType.ALARM_DELETE, null);
             sendEntityNotificationMsg(getTenantId(), alarmId, EdgeEventActionType.DELETED);
 
             return alarmService.deleteAlarm(getTenantId(), alarmId);
@@ -177,7 +178,6 @@ public class AlarmController extends BaseController {
             @RequestParam(required = false) String sortOrder,
             @RequestParam(required = false) Long startTime,
             @RequestParam(required = false) Long endTime,
-            @RequestParam(required = false) String offset,
             @RequestParam(required = false) Boolean fetchOriginator
     ) throws ThingsboardException {
         checkParameter("EntityId", strEntityId);
@@ -191,12 +191,43 @@ public class AlarmController extends BaseController {
         }
         checkEntityId(entityId, Operation.READ);
         TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime);
-        UUID idOffsetUuid = null;
-        if (StringUtils.isNotEmpty(offset)) {
-            idOffsetUuid = toUUID(offset);
+
+        try {
+            return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get());
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
+
+    @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
+    @RequestMapping(value = "/alarms", method = RequestMethod.GET)
+    @ResponseBody
+    public PageData<AlarmInfo> getAllAlarms(
+            @RequestParam(required = false) String searchStatus,
+            @RequestParam(required = false) String status,
+            @RequestParam int pageSize,
+            @RequestParam int page,
+            @RequestParam(required = false) String textSearch,
+            @RequestParam(required = false) String sortProperty,
+            @RequestParam(required = false) String sortOrder,
+            @RequestParam(required = false) Long startTime,
+            @RequestParam(required = false) Long endTime,
+            @RequestParam(required = false) Boolean fetchOriginator
+    ) throws ThingsboardException {
+        AlarmSearchStatus alarmSearchStatus = StringUtils.isEmpty(searchStatus) ? null : AlarmSearchStatus.valueOf(searchStatus);
+        AlarmStatus alarmStatus = StringUtils.isEmpty(status) ? null : AlarmStatus.valueOf(status);
+        if (alarmSearchStatus != null && alarmStatus != null) {
+            throw new ThingsboardException("Invalid alarms search query: Both parameters 'searchStatus' " +
+                    "and 'status' can't be specified at the same time!", ThingsboardErrorCode.BAD_REQUEST_PARAMS);
         }
+        TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime);
+
         try {
-            return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator, idOffsetUuid)).get());
+            if (getCurrentUser().isCustomerUser()) {
+                return checkNotNull(alarmService.findCustomerAlarms(getCurrentUser().getTenantId(), getCurrentUser().getCustomerId(), new AlarmQuery(null, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get());
+            } else {
+                return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(null, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get());
+            }
         } catch (Exception e) {
             throw handleException(e);
         }

application/src/main/java/org/thingsboard/server/controller/BaseController.java

@@ -17,20 +17,19 @@ package org.thingsboard.server.controller;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.MediaType;
 import org.springframework.security.core.Authentication;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.thingsboard.server.common.data.Customer;
 import org.thingsboard.server.common.data.Dashboard;
 import org.thingsboard.server.common.data.DashboardInfo;
-import org.thingsboard.server.common.data.DataConstants;
 import org.thingsboard.server.common.data.Device;
 import org.thingsboard.server.common.data.DeviceInfo;
 import org.thingsboard.server.common.data.DeviceProfile;
@@ -38,6 +37,8 @@ import org.thingsboard.server.common.data.EdgeUtils;
 import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.EntityView;
 import org.thingsboard.server.common.data.EntityViewInfo;
+import org.thingsboard.server.common.data.OtaPackage;
+import org.thingsboard.server.common.data.OtaPackageInfo;
 import org.thingsboard.server.common.data.HasName;
 import org.thingsboard.server.common.data.HasTenantId;
 import org.thingsboard.server.common.data.TbResourceInfo;
@@ -67,6 +68,8 @@ import org.thingsboard.server.common.data.id.EdgeId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.EntityIdFactory;
 import org.thingsboard.server.common.data.id.EntityViewId;
+import org.thingsboard.server.common.data.id.OtaPackageId;
+import org.thingsboard.server.common.data.id.RpcId;
 import org.thingsboard.server.common.data.id.TbResourceId;
 import org.thingsboard.server.common.data.id.RuleChainId;
 import org.thingsboard.server.common.data.id.RuleNodeId;
@@ -75,24 +78,18 @@ import org.thingsboard.server.common.data.id.TenantProfileId;
 import org.thingsboard.server.common.data.id.UserId;
 import org.thingsboard.server.common.data.id.WidgetTypeId;
 import org.thingsboard.server.common.data.id.WidgetsBundleId;
-import org.thingsboard.server.common.data.kv.AttributeKvEntry;
-import org.thingsboard.server.common.data.kv.DataType;
-import org.thingsboard.server.common.data.kv.KvEntry;
-import org.thingsboard.server.common.data.kv.TsKvEntry;
 import org.thingsboard.server.common.data.page.PageLink;
 import org.thingsboard.server.common.data.page.SortOrder;
 import org.thingsboard.server.common.data.page.TimePageLink;
 import org.thingsboard.server.common.data.plugin.ComponentDescriptor;
 import org.thingsboard.server.common.data.plugin.ComponentType;
 import org.thingsboard.server.common.data.relation.EntityRelation;
+import org.thingsboard.server.common.data.rpc.Rpc;
 import org.thingsboard.server.common.data.rule.RuleChain;
 import org.thingsboard.server.common.data.rule.RuleChainType;
 import org.thingsboard.server.common.data.rule.RuleNode;
 import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
 import org.thingsboard.server.common.data.widget.WidgetsBundle;
-import org.thingsboard.server.common.msg.TbMsg;
-import org.thingsboard.server.common.msg.TbMsgDataType;
-import org.thingsboard.server.common.msg.TbMsgMetaData;
 import org.thingsboard.server.dao.asset.AssetService;
 import org.thingsboard.server.dao.attributes.AttributesService;
 import org.thingsboard.server.dao.audit.AuditLogService;
@@ -106,11 +103,12 @@ import org.thingsboard.server.dao.edge.EdgeService;
 import org.thingsboard.server.dao.entityview.EntityViewService;
 import org.thingsboard.server.dao.exception.DataValidationException;
 import org.thingsboard.server.dao.exception.IncorrectParameterException;
+import org.thingsboard.server.dao.ota.OtaPackageService;
 import org.thingsboard.server.dao.model.ModelConstants;
 import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
 import org.thingsboard.server.dao.oauth2.OAuth2Service;
 import org.thingsboard.server.dao.relation.RelationService;
-import org.thingsboard.server.dao.resource.TbResourceService;
+import org.thingsboard.server.dao.rpc.RpcService;
 import org.thingsboard.server.dao.rule.RuleChainService;
 import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
 import org.thingsboard.server.dao.tenant.TenantProfileService;
@@ -123,13 +121,16 @@ import org.thingsboard.server.gen.transport.TransportProtos;
 import org.thingsboard.server.queue.discovery.PartitionService;
 import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
 import org.thingsboard.server.queue.util.TbCoreComponent;
+import org.thingsboard.server.service.action.RuleEngineEntityActionService;
 import org.thingsboard.server.service.component.ComponentDiscoveryService;
+import org.thingsboard.server.service.ota.OtaPackageStateService;
 import org.thingsboard.server.service.edge.EdgeNotificationService;
 import org.thingsboard.server.service.edge.rpc.EdgeGrpcService;
 import org.thingsboard.server.service.edge.rpc.init.SyncEdgeService;
-import org.thingsboard.server.service.lwm2m.LwM2MModelsRepository;
+import org.thingsboard.server.service.lwm2m.LwM2MServerSecurityInfoRepository;
 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
 import org.thingsboard.server.service.queue.TbClusterService;
+import org.thingsboard.server.service.resource.TbResourceService;
 import org.thingsboard.server.service.security.model.SecurityUser;
 import org.thingsboard.server.service.security.permission.AccessControlService;
 import org.thingsboard.server.service.security.permission.Operation;
@@ -141,11 +142,9 @@ import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
 import javax.mail.MessagingException;
 import javax.servlet.http.HttpServletResponse;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
-import java.util.stream.Collectors;
 
 import static org.thingsboard.server.dao.service.Validator.validateId;
 
@@ -154,8 +153,6 @@ import static org.thingsboard.server.dao.service.Validator.validateId;
 public abstract class BaseController {
 
     public static final String INCORRECT_TENANT_ID = "Incorrect tenantId ";
-    public static final String YOU_DON_T_HAVE_PERMISSION_TO_PERFORM_THIS_OPERATION = "You don't have permission to perform this operation!";
-
     protected static final String DEFAULT_DASHBOARD = "defaultDashboardId";
     protected static final String HOME_DASHBOARD = "homeDashboardId";
 
@@ -245,6 +242,15 @@ public abstract class BaseController {
     @Autowired
     protected TbResourceService resourceService;
 
+    @Autowired
+    protected OtaPackageService otaPackageService;
+
+    @Autowired
+    protected OtaPackageStateService otaPackageStateService;
+
+    @Autowired
+    protected RpcService rpcService;
+
     @Autowired
     protected TbQueueProducerProvider producerProvider;
 
@@ -255,7 +261,7 @@ public abstract class BaseController {
     protected TbDeviceProfileCache deviceProfileCache;
 
     @Autowired
-    protected LwM2MModelsRepository lwM2MModelsRepository;
+    protected LwM2MServerSecurityInfoRepository lwM2MServerSecurityInfoRepository;
 
     @Autowired(required = false)
     protected EdgeService edgeService;
@@ -269,6 +275,9 @@ public abstract class BaseController {
     @Autowired(required = false)
     protected EdgeGrpcService edgeGrpcService;
 
+    @Autowired
+    protected RuleEngineEntityActionService ruleEngineEntityActionService;
+
     @Value("${server.log_controller_error_stack_trace}")
     @Getter
     private boolean logControllerErrorStackTrace;
@@ -501,6 +510,9 @@ public abstract class BaseController {
                 case TB_RESOURCE:
                     checkResourceId(new TbResourceId(entityId.getId()), operation);
                     return;
+                case OTA_PACKAGE:
+                    checkOtaPackageId(new OtaPackageId(entityId.getId()), operation);
+                    return;
                 default:
                     throw new IllegalArgumentException("Unsupported entity type: " + entityId.getEntityType());
             }
@@ -756,6 +768,42 @@ public abstract class BaseController {
         }
     }
 
+    OtaPackage checkOtaPackageId(OtaPackageId otaPackageId, Operation operation) throws ThingsboardException {
+        try {
+            validateId(otaPackageId, "Incorrect otaPackageId " + otaPackageId);
+            OtaPackage otaPackage = otaPackageService.findOtaPackageById(getCurrentUser().getTenantId(), otaPackageId);
+            checkNotNull(otaPackage);
+            accessControlService.checkPermission(getCurrentUser(), Resource.OTA_PACKAGE, operation, otaPackageId, otaPackage);
+            return otaPackage;
+        } catch (Exception e) {
+            throw handleException(e, false);
+        }
+    }
+
+    OtaPackageInfo checkOtaPackageInfoId(OtaPackageId otaPackageId, Operation operation) throws ThingsboardException {
+        try {
+            validateId(otaPackageId, "Incorrect otaPackageId " + otaPackageId);
+            OtaPackageInfo otaPackageIn = otaPackageService.findOtaPackageInfoById(getCurrentUser().getTenantId(), otaPackageId);
+            checkNotNull(otaPackageIn);
+            accessControlService.checkPermission(getCurrentUser(), Resource.OTA_PACKAGE, operation, otaPackageId, otaPackageIn);
+            return otaPackageIn;
+        } catch (Exception e) {
+            throw handleException(e, false);
+        }
+    }
+
+    Rpc checkRpcId(RpcId rpcId, Operation operation) throws ThingsboardException {
+        try {
+            validateId(rpcId, "Incorrect rpcId " + rpcId);
+            Rpc rpc = rpcService.findById(getCurrentUser().getTenantId(), rpcId);
+            checkNotNull(rpc);
+            accessControlService.checkPermission(getCurrentUser(), Resource.RPC, operation, rpcId, rpc);
+            return rpc;
+        } catch (Exception e) {
+            throw handleException(e, false);
+        }
+    }
+
     @SuppressWarnings("unchecked")
     protected <I extends EntityId> I emptyId(EntityType entityType) {
         return (I) EntityIdFactory.getByTypeAndUuid(entityType, ModelConstants.NULL_UUID);
@@ -772,7 +820,7 @@ public abstract class BaseController {
             customerId = user.getCustomerId();
         }
         if (e == null) {
-            pushEntityActionToRuleEngine(entityId, entity, user, customerId, actionType, additionalInfo);
+            ruleEngineEntityActionService.pushEntityActionToRuleEngine(entityId, entity, user.getTenantId(), customerId, actionType, user, additionalInfo);
         }
         auditLogService.logEntityAction(user.getTenantId(), customerId, user.getId(), user.getName(), entityId, entity, actionType, e, additionalInfo);
     }
@@ -782,184 +830,6 @@ public abstract class BaseController {
         return error != null ? (Exception.class.isInstance(error) ? (Exception) error : new Exception(error)) : null;
     }
 
-    private <E extends HasName, I extends EntityId> void pushEntityActionToRuleEngine(I entityId, E entity, User user, CustomerId customerId,
-                                                                                      ActionType actionType, Object... additionalInfo) {
-        String msgType = null;
-        switch (actionType) {
-            case ADDED:
-                msgType = DataConstants.ENTITY_CREATED;
-                break;
-            case DELETED:
-                msgType = DataConstants.ENTITY_DELETED;
-                break;
-            case UPDATED:
-                msgType = DataConstants.ENTITY_UPDATED;
-                break;
-            case ASSIGNED_TO_CUSTOMER:
-                msgType = DataConstants.ENTITY_ASSIGNED;
-                break;
-            case UNASSIGNED_FROM_CUSTOMER:
-                msgType = DataConstants.ENTITY_UNASSIGNED;
-                break;
-            case ATTRIBUTES_UPDATED:
-                msgType = DataConstants.ATTRIBUTES_UPDATED;
-                break;
-            case ATTRIBUTES_DELETED:
-                msgType = DataConstants.ATTRIBUTES_DELETED;
-                break;
-            case ALARM_ACK:
-                msgType = DataConstants.ALARM_ACK;
-                break;
-            case ALARM_CLEAR:
-                msgType = DataConstants.ALARM_CLEAR;
-                break;
-            case ASSIGNED_FROM_TENANT:
-                msgType = DataConstants.ENTITY_ASSIGNED_FROM_TENANT;
-                break;
-            case ASSIGNED_TO_TENANT:
-                msgType = DataConstants.ENTITY_ASSIGNED_TO_TENANT;
-                break;
-            case PROVISION_SUCCESS:
-                msgType = DataConstants.PROVISION_SUCCESS;
-                break;
-            case PROVISION_FAILURE:
-                msgType = DataConstants.PROVISION_FAILURE;
-                break;
-            case TIMESERIES_UPDATED:
-                msgType = DataConstants.TIMESERIES_UPDATED;
-                break;
-            case TIMESERIES_DELETED:
-                msgType = DataConstants.TIMESERIES_DELETED;
-                break;
-            case ASSIGNED_TO_EDGE:
-                msgType = DataConstants.ENTITY_ASSIGNED_TO_EDGE;
-                break;
-            case UNASSIGNED_FROM_EDGE:
-                msgType = DataConstants.ENTITY_UNASSIGNED_FROM_EDGE;
-                break;
-        }
-        if (!StringUtils.isEmpty(msgType)) {
-            try {
-                TbMsgMetaData metaData = new TbMsgMetaData();
-                metaData.putValue("userId", user.getId().toString());
-                metaData.putValue("userName", user.getName());
-                if (customerId != null && !customerId.isNullUid()) {
-                    metaData.putValue("customerId", customerId.toString());
-                }
-                if (actionType == ActionType.ASSIGNED_TO_CUSTOMER) {
-                    String strCustomerId = extractParameter(String.class, 1, additionalInfo);
-                    String strCustomerName = extractParameter(String.class, 2, additionalInfo);
-                    metaData.putValue("assignedCustomerId", strCustomerId);
-                    metaData.putValue("assignedCustomerName", strCustomerName);
-                } else if (actionType == ActionType.UNASSIGNED_FROM_CUSTOMER) {
-                    String strCustomerId = extractParameter(String.class, 1, additionalInfo);
-                    String strCustomerName = extractParameter(String.class, 2, additionalInfo);
-                    metaData.putValue("unassignedCustomerId", strCustomerId);
-                    metaData.putValue("unassignedCustomerName", strCustomerName);
-                } else if (actionType == ActionType.ASSIGNED_FROM_TENANT) {
-                    String strTenantId = extractParameter(String.class, 0, additionalInfo);
-                    String strTenantName = extractParameter(String.class, 1, additionalInfo);
-                    metaData.putValue("assignedFromTenantId", strTenantId);
-                    metaData.putValue("assignedFromTenantName", strTenantName);
-                } else if (actionType == ActionType.ASSIGNED_TO_TENANT) {
-                    String strTenantId = extractParameter(String.class, 0, additionalInfo);
-                    String strTenantName = extractParameter(String.class, 1, additionalInfo);
-                    metaData.putValue("assignedToTenantId", strTenantId);
-                    metaData.putValue("assignedToTenantName", strTenantName);
-                } else if (actionType == ActionType.ASSIGNED_TO_EDGE) {
-                    String strEdgeId = extractParameter(String.class, 1, additionalInfo);
-                    String strEdgeName = extractParameter(String.class, 2, additionalInfo);
-                    metaData.putValue("assignedEdgeId", strEdgeId);
-                    metaData.putValue("assignedEdgeName", strEdgeName);
-                } else if (actionType == ActionType.UNASSIGNED_FROM_EDGE) {
-                    String strEdgeId = extractParameter(String.class, 1, additionalInfo);
-                    String strEdgeName = extractParameter(String.class, 2, additionalInfo);
-                    metaData.putValue("unassignedEdgeId", strEdgeId);
-                    metaData.putValue("unassignedEdgeName", strEdgeName);
-                }
-                ObjectNode entityNode;
-                if (entity != null) {
-                    entityNode = json.valueToTree(entity);
-                    if (entityId.getEntityType() == EntityType.DASHBOARD) {
-                        entityNode.put("configuration", "");
-                    }
-                } else {
-                    entityNode = json.createObjectNode();
-                    if (actionType == ActionType.ATTRIBUTES_UPDATED) {
-                        String scope = extractParameter(String.class, 0, additionalInfo);
-                        @SuppressWarnings("unchecked")
-                        List<AttributeKvEntry> attributes = extractParameter(List.class, 1, additionalInfo);
-                        metaData.putValue(DataConstants.SCOPE, scope);
-                        if (attributes != null) {
-                            for (AttributeKvEntry attr : attributes) {
-                                addKvEntry(entityNode, attr);
-                            }
-                        }
-                    } else if (actionType == ActionType.ATTRIBUTES_DELETED) {
-                        String scope = extractParameter(String.class, 0, additionalInfo);
-                        @SuppressWarnings("unchecked")
-                        List<String> keys = extractParameter(List.class, 1, additionalInfo);
-                        metaData.putValue(DataConstants.SCOPE, scope);
-                        ArrayNode attrsArrayNode = entityNode.putArray("attributes");
-                        if (keys != null) {
-                            keys.forEach(attrsArrayNode::add);
-                        }
-                    } else if (actionType == ActionType.TIMESERIES_UPDATED) {
-                        @SuppressWarnings("unchecked")
-                        List<TsKvEntry> timeseries = extractParameter(List.class, 0, additionalInfo);
-                        addTimeseries(entityNode, timeseries);
-                    } else if (actionType == ActionType.TIMESERIES_DELETED) {
-                        @SuppressWarnings("unchecked")
-                        List<String> keys = extractParameter(List.class, 0, additionalInfo);
-                        if (keys != null) {
-                            ArrayNode timeseriesArrayNode = entityNode.putArray("timeseries");
-                            keys.forEach(timeseriesArrayNode::add);
-                        }
-                        entityNode.put("startTs", extractParameter(Long.class, 1, additionalInfo));
-                        entityNode.put("endTs", extractParameter(Long.class, 2, additionalInfo));
-                    }
-                }
-                TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode));
-                TenantId tenantId = user.getTenantId();
-                if (tenantId.isNullUid()) {
-                    if (entity instanceof HasTenantId) {
-                        tenantId = ((HasTenantId) entity).getTenantId();
-                    }
-                }
-                tbClusterService.pushMsgToRuleEngine(tenantId, entityId, tbMsg, null);
-            } catch (Exception e) {
-                log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e);
-            }
-        }
-    }
-
-    private void addKvEntry(ObjectNode entityNode, KvEntry kvEntry) throws Exception {
-        if (kvEntry.getDataType() == DataType.BOOLEAN) {
-            kvEntry.getBooleanValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
-        } else if (kvEntry.getDataType() == DataType.DOUBLE) {
-            kvEntry.getDoubleValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
-        } else if (kvEntry.getDataType() == DataType.LONG) {
-            kvEntry.getLongValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
-        } else if (kvEntry.getDataType() == DataType.JSON) {
-            if (kvEntry.getJsonValue().isPresent()) {
-                entityNode.set(kvEntry.getKey(), json.readTree(kvEntry.getJsonValue().get()));
-            }
-        } else {
-            entityNode.put(kvEntry.getKey(), kvEntry.getValueAsString());
-        }
-    }
-
-    private <T> T extractParameter(Class<T> clazz, int index, Object... additionalInfo) {
-        T result = null;
-        if (additionalInfo != null && additionalInfo.length > index) {
-            Object paramObject = additionalInfo[index];
-            if (clazz.isInstance(paramObject)) {
-                result = clazz.cast(paramObject);
-            }
-        }
-        return result;
-    }
-
     protected <E extends HasName> String entityToStr(E entity) {
         try {
             return json.writeValueAsString(json.valueToTree(entity));
@@ -1056,23 +926,6 @@ public abstract class BaseController {
         return result;
     }
 
-    private void addTimeseries(ObjectNode entityNode, List<TsKvEntry> timeseries) throws Exception {
-        if (timeseries != null && !timeseries.isEmpty()) {
-            ArrayNode result = entityNode.putArray("timeseries");
-            Map<Long, List<TsKvEntry>> groupedTelemetry = timeseries.stream()
-                    .collect(Collectors.groupingBy(TsKvEntry::getTs));
-            for (Map.Entry<Long, List<TsKvEntry>> entry : groupedTelemetry.entrySet()) {
-                ObjectNode element = json.createObjectNode();
-                element.put("ts", entry.getKey());
-                ObjectNode values = element.putObject("values");
-                for (TsKvEntry tsKvEntry : entry.getValue()) {
-                    addKvEntry(values, tsKvEntry);
-                }
-                result.add(element);
-            }
-        }
-    }
-
     protected void processDashboardIdFromAdditionalInfo(ObjectNode additionalInfo, String requiredFields) throws ThingsboardException {
         String dashboardId = additionalInfo.has(requiredFields) ? additionalInfo.get(requiredFields).asText() : null;
         if (dashboardId != null && !dashboardId.equals("null")) {
@@ -1082,4 +935,11 @@ public abstract class BaseController {
         }
     }
 
+    protected MediaType parseMediaType(String contentType) {
+        try {
+            return MediaType.parseMediaType(contentType);
+        } catch (Exception e) {
+            return MediaType.APPLICATION_OCTET_STREAM;
+        }
+    }
 }

application/src/main/java/org/thingsboard/server/controller/CustomerController.java

@@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.id.EdgeId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.page.PageLink;
+import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.security.permission.Operation;
 import org.thingsboard.server.service.security.permission.Resource;
@@ -148,6 +149,7 @@ public class CustomerController extends BaseController {
                     ActionType.DELETED, null, strCustomerId);
 
             sendDeleteNotificationMsg(getTenantId(), customerId, relatedEdgeIds);
+            tbClusterService.onEntityStateChange(getTenantId(), customerId, ComponentLifecycleEvent.DELETED);
         } catch (Exception e) {
 
             logEntityAction(emptyId(EntityType.CUSTOMER),

application/src/main/java/org/thingsboard/server/controller/DashboardController.java

@@ -533,6 +533,37 @@ public class DashboardController extends BaseController {
         }
     }
 
+    @PreAuthorize("isAuthenticated()")
+    @RequestMapping(value = "/dashboard/home/info", method = RequestMethod.GET)
+    @ResponseBody
+    public HomeDashboardInfo getHomeDashboardInfo() throws ThingsboardException {
+        try {
+            SecurityUser securityUser = getCurrentUser();
+            if (securityUser.isSystemAdmin()) {
+                return null;
+            }
+            User user = userService.findUserById(securityUser.getTenantId(), securityUser.getId());
+            JsonNode additionalInfo = user.getAdditionalInfo();
+            HomeDashboardInfo homeDashboardInfo;
+            homeDashboardInfo = extractHomeDashboardInfoFromAdditionalInfo(additionalInfo);
+            if (homeDashboardInfo == null) {
+                if (securityUser.isCustomerUser()) {
+                    Customer customer = customerService.findCustomerById(securityUser.getTenantId(), securityUser.getCustomerId());
+                    additionalInfo = customer.getAdditionalInfo();
+                    homeDashboardInfo = extractHomeDashboardInfoFromAdditionalInfo(additionalInfo);
+                }
+                if (homeDashboardInfo == null) {
+                    Tenant tenant = tenantService.findTenantById(securityUser.getTenantId());
+                    additionalInfo = tenant.getAdditionalInfo();
+                    homeDashboardInfo = extractHomeDashboardInfoFromAdditionalInfo(additionalInfo);
+                }
+            }
+            return homeDashboardInfo;
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
+
     @PreAuthorize("hasAuthority('TENANT_ADMIN')")
     @RequestMapping(value = "/tenant/dashboard/home/info", method = RequestMethod.GET)
     @ResponseBody
@@ -582,6 +613,22 @@ public class DashboardController extends BaseController {
         }
     }
 
+    private HomeDashboardInfo extractHomeDashboardInfoFromAdditionalInfo(JsonNode additionalInfo) {
+        try {
+            if (additionalInfo != null && additionalInfo.has(HOME_DASHBOARD_ID) && !additionalInfo.get(HOME_DASHBOARD_ID).isNull()) {
+                String strDashboardId = additionalInfo.get(HOME_DASHBOARD_ID).asText();
+                DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
+                checkDashboardId(dashboardId, Operation.READ);
+                boolean hideDashboardToolbar = true;
+                if (additionalInfo.has(HOME_DASHBOARD_HIDE_TOOLBAR)) {
+                    hideDashboardToolbar = additionalInfo.get(HOME_DASHBOARD_HIDE_TOOLBAR).asBoolean();
+                }
+                return new HomeDashboardInfo(dashboardId, hideDashboardToolbar);
+            }
+        } catch (Exception e) {}
+        return null;
+    }
+
     private HomeDashboard extractHomeDashboardFromAdditionalInfo(JsonNode additionalInfo) {
         try {
             if (additionalInfo != null && additionalInfo.has(HOME_DASHBOARD_ID) && !additionalInfo.get(HOME_DASHBOARD_ID).isNull()) {

application/src/main/java/org/thingsboard/server/controller/DeviceController.java

@@ -53,6 +53,7 @@ import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.DeviceProfileId;
 import org.thingsboard.server.common.data.id.EdgeId;
 import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.ota.OtaPackageType;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.page.PageLink;
 import org.thingsboard.server.common.data.page.TimePageLink;
@@ -75,6 +76,7 @@ import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.UUID;
 import java.util.stream.Collectors;
 
 import static org.thingsboard.server.controller.EdgeController.EDGE_ID;
@@ -119,38 +121,49 @@ public class DeviceController extends BaseController {
     @ResponseBody
     public Device saveDevice(@RequestBody Device device,
                              @RequestParam(name = "accessToken", required = false) String accessToken) throws ThingsboardException {
+        boolean created = device.getId() == null;
         try {
             device.setTenantId(getCurrentUser().getTenantId());
 
             checkEntity(device.getId(), device, Resource.DEVICE);
 
+            Device oldDevice;
+            if (!created) {
+                oldDevice = deviceService.findDeviceById(getTenantId(), device.getId());
+            } else {
+                oldDevice = null;
+            }
+
             Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken));
 
             tbClusterService.onDeviceChange(savedDevice, null);
             tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(),
                     savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null);
-            tbClusterService.onEntityStateChange(savedDevice.getTenantId(), savedDevice.getId(),
-                    device.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
+            tbClusterService.onEntityStateChange(savedDevice.getTenantId(), savedDevice.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
 
-            if (device.getId() != null) {
+            if (!created) {
                 sendEntityNotificationMsg(savedDevice.getTenantId(), savedDevice.getId(), EdgeEventActionType.UPDATED);
             }
 
             logEntityAction(savedDevice.getId(), savedDevice,
                     savedDevice.getCustomerId(),
-                    device.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null);
+                    created ? ActionType.ADDED : ActionType.UPDATED, null);
 
             if (device.getId() == null) {
                 deviceStateService.onDeviceAdded(savedDevice);
             } else {
                 deviceStateService.onDeviceUpdated(savedDevice);
             }
+
+            otaPackageStateService.update(savedDevice, oldDevice);
+
             return savedDevice;
         } catch (Exception e) {
             logEntityAction(emptyId(EntityType.DEVICE), device,
-                    null, device.getId() == null ? ActionType.ADDED : ActionType.UPDATED, e);
+                    null, created ? ActionType.ADDED : ActionType.UPDATED, e);
             throw handleException(e);
         }
+
     }
 
     @PreAuthorize("hasAuthority('TENANT_ADMIN')")
@@ -645,7 +658,7 @@ public class DeviceController extends BaseController {
     private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) {
         String data = entityToStr(assignedDevice);
         if (data != null) {
-            TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_ASSIGNED_FROM_TENANT, assignedDevice.getId(), getMetaDataForAssignedFrom(currentTenant), TbMsgDataType.JSON, data);
+            TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_ASSIGNED_FROM_TENANT, assignedDevice.getId(), assignedDevice.getCustomerId(), getMetaDataForAssignedFrom(currentTenant), TbMsgDataType.JSON, data);
             tbClusterService.pushMsgToRuleEngine(newTenantId, assignedDevice.getId(), tbMsg, null);
         }
     }
@@ -767,4 +780,21 @@ public class DeviceController extends BaseController {
             throw handleException(e);
         }
     }
+
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
+    @RequestMapping(value = "/devices/count/{otaPackageType}/{deviceProfileId}", method = RequestMethod.GET)
+    @ResponseBody
+    public Long countByDeviceProfileAndEmptyOtaPackage(@PathVariable("otaPackageType") String otaPackageType,
+                                                       @PathVariable("deviceProfileId") String deviceProfileId) throws ThingsboardException {
+        checkParameter("OtaPackageType", otaPackageType);
+        checkParameter("DeviceProfileId", deviceProfileId);
+        try {
+            return deviceService.countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage(
+                    getTenantId(),
+                    new DeviceProfileId(UUID.fromString(deviceProfileId)),
+                    OtaPackageType.valueOf(otaPackageType));
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
 }

application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java

@@ -44,6 +44,7 @@ import org.thingsboard.server.service.security.permission.Operation;
 import org.thingsboard.server.service.security.permission.Resource;
 
 import java.util.List;
+import java.util.Objects;
 import java.util.UUID;
 
 @RestController
@@ -144,6 +145,19 @@ public class DeviceProfileController extends BaseController {
 
             checkEntity(deviceProfile.getId(), deviceProfile, Resource.DEVICE_PROFILE);
 
+            boolean isFirmwareChanged = false;
+            boolean isSoftwareChanged = false;
+
+            if (!created) {
+                DeviceProfile oldDeviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), deviceProfile.getId());
+                if (!Objects.equals(deviceProfile.getFirmwareId(), oldDeviceProfile.getFirmwareId())) {
+                    isFirmwareChanged = true;
+                }
+                if (!Objects.equals(deviceProfile.getSoftwareId(), oldDeviceProfile.getSoftwareId())) {
+                    isSoftwareChanged = true;
+                }
+            }
+
             DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile));
 
             tbClusterService.onDeviceProfileChange(savedDeviceProfile, null);
@@ -154,9 +168,10 @@ public class DeviceProfileController extends BaseController {
                     null,
                     created ? ActionType.ADDED : ActionType.UPDATED, null);
 
+            otaPackageStateService.update(savedDeviceProfile, isFirmwareChanged, isSoftwareChanged);
+
             sendEntityNotificationMsg(getTenantId(), savedDeviceProfile.getId(),
                     deviceProfile.getId() == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED);
-
             return savedDeviceProfile;
         } catch (Exception e) {
             logEntityAction(emptyId(EntityType.DEVICE_PROFILE), deviceProfile,

application/src/main/java/org/thingsboard/server/controller/Lwm2mController.java

@@ -44,12 +44,11 @@ import java.util.Map;
 public class Lwm2mController extends BaseController {
 
     @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
-    @RequestMapping(value = "/lwm2m/deviceProfile/bootstrap/{securityMode}/{bootstrapServerIs}", method = RequestMethod.GET)
+    @RequestMapping(value = "/lwm2m/deviceProfile/bootstrap/{isBootstrapServer}", method = RequestMethod.GET)
     @ResponseBody
-    public ServerSecurityConfig getLwm2mBootstrapSecurityInfo(@PathVariable("securityMode") String securityMode,
-                                                              @PathVariable("bootstrapServerIs") boolean bootstrapServerIs) throws ThingsboardException {
+    public ServerSecurityConfig getLwm2mBootstrapSecurityInfo(@PathVariable("isBootstrapServer") boolean bootstrapServer) throws ThingsboardException {
         try {
-            return lwM2MModelsRepository.getBootstrapSecurityInfo(securityMode, bootstrapServerIs);
+            return lwM2MServerSecurityInfoRepository.getServerSecurityInfo(bootstrapServer);
         } catch (Exception e) {
             throw handleException(e);
         }

application/src/main/java/org/thingsboard/server/controller/OAuth2Controller.java

@@ -22,12 +22,15 @@ import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestController;
+import org.thingsboard.server.common.data.StringUtils;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo;
-import org.thingsboard.server.common.data.oauth2.OAuth2ClientsParams;
+import org.thingsboard.server.common.data.oauth2.OAuth2Info;
+import org.thingsboard.server.common.data.oauth2.PlatformType;
 import org.thingsboard.server.dao.oauth2.OAuth2Configuration;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.security.permission.Operation;
@@ -49,7 +52,9 @@ public class OAuth2Controller extends BaseController {
 
     @RequestMapping(value = "/noauth/oauth2Clients", method = RequestMethod.POST)
     @ResponseBody
-    public List<OAuth2ClientInfo> getOAuth2Clients(HttpServletRequest request) throws ThingsboardException {
+    public List<OAuth2ClientInfo> getOAuth2Clients(HttpServletRequest request,
+                                                   @RequestParam(required = false) String pkgName,
+                                                   @RequestParam(required = false) String platform) throws ThingsboardException {
         try {
             if (log.isDebugEnabled()) {
                 log.debug("Executing getOAuth2Clients: [{}][{}][{}]", request.getScheme(), request.getServerName(), request.getServerPort());
@@ -59,7 +64,13 @@ public class OAuth2Controller extends BaseController {
                     log.debug("Header: {} {}", header, request.getHeader(header));
                 }
             }
-            return oAuth2Service.getOAuth2Clients(MiscUtils.getScheme(request), MiscUtils.getDomainNameAndPort(request));
+            PlatformType platformType = null;
+            if (StringUtils.isNotEmpty(platform)) {
+                try {
+                    platformType = PlatformType.valueOf(platform);
+                } catch (Exception e) {}
+            }
+            return oAuth2Service.getOAuth2Clients(MiscUtils.getScheme(request), MiscUtils.getDomainNameAndPort(request), pkgName, platformType);
         } catch (Exception e) {
             throw handleException(e);
         }
@@ -68,10 +79,10 @@ public class OAuth2Controller extends BaseController {
     @PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
     @RequestMapping(value = "/oauth2/config", method = RequestMethod.GET, produces = "application/json")
     @ResponseBody
-    public OAuth2ClientsParams getCurrentOAuth2Params() throws ThingsboardException {
+    public OAuth2Info getCurrentOAuth2Info() throws ThingsboardException {
         try {
             accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_INFO, Operation.READ);
-            return oAuth2Service.findOAuth2Params();
+            return oAuth2Service.findOAuth2Info();
         } catch (Exception e) {
             throw handleException(e);
         }
@@ -80,11 +91,11 @@ public class OAuth2Controller extends BaseController {
     @PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
     @RequestMapping(value = "/oauth2/config", method = RequestMethod.POST)
     @ResponseStatus(value = HttpStatus.OK)
-    public OAuth2ClientsParams saveOAuth2Params(@RequestBody OAuth2ClientsParams oauth2Params) throws ThingsboardException {
+    public OAuth2Info saveOAuth2Info(@RequestBody OAuth2Info oauth2Info) throws ThingsboardException {
         try {
             accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_INFO, Operation.WRITE);
-            oAuth2Service.saveOAuth2Params(oauth2Params);
-            return oAuth2Service.findOAuth2Params();
+            oAuth2Service.saveOAuth2Info(oauth2Info);
+            return oAuth2Service.findOAuth2Info();
         } catch (Exception e) {
             throw handleException(e);
         }

application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java

@@ -0,0 +1,225 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.controller;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.core.io.ByteArrayResource;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.OtaPackage;
+import org.thingsboard.server.common.data.OtaPackageInfo;
+import org.thingsboard.server.common.data.audit.ActionType;
+import org.thingsboard.server.common.data.exception.ThingsboardException;
+import org.thingsboard.server.common.data.ota.ChecksumAlgorithm;
+import org.thingsboard.server.common.data.ota.OtaPackageType;
+import org.thingsboard.server.common.data.id.DeviceProfileId;
+import org.thingsboard.server.common.data.id.OtaPackageId;
+import org.thingsboard.server.common.data.page.PageData;
+import org.thingsboard.server.common.data.page.PageLink;
+import org.thingsboard.server.queue.util.TbCoreComponent;
+import org.thingsboard.server.service.security.permission.Operation;
+import org.thingsboard.server.service.security.permission.Resource;
+
+import java.nio.ByteBuffer;
+
+@Slf4j
+@RestController
+@TbCoreComponent
+@RequestMapping("/api")
+public class OtaPackageController extends BaseController {
+
+    public static final String OTA_PACKAGE_ID = "otaPackageId";
+    public static final String CHECKSUM_ALGORITHM = "checksumAlgorithm";
+
+    @PreAuthorize("hasAnyAuthority( 'TENANT_ADMIN')")
+    @RequestMapping(value = "/otaPackage/{otaPackageId}/download", method = RequestMethod.GET)
+    @ResponseBody
+    public ResponseEntity<org.springframework.core.io.Resource> downloadOtaPackage(@PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException {
+        checkParameter(OTA_PACKAGE_ID, strOtaPackageId);
+        try {
+            OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId));
+            OtaPackage otaPackage = checkOtaPackageId(otaPackageId, Operation.READ);
+
+            if (otaPackage.hasUrl()) {
+                return ResponseEntity.badRequest().build();
+            }
+
+            ByteArrayResource resource = new ByteArrayResource(otaPackage.getData().array());
+            return ResponseEntity.ok()
+                    .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + otaPackage.getFileName())
+                    .header("x-filename", otaPackage.getFileName())
+                    .contentLength(resource.contentLength())
+                    .contentType(parseMediaType(otaPackage.getContentType()))
+                    .body(resource);
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
+
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
+    @RequestMapping(value = "/otaPackage/info/{otaPackageId}", method = RequestMethod.GET)
+    @ResponseBody
+    public OtaPackageInfo getOtaPackageInfoById(@PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException {
+        checkParameter(OTA_PACKAGE_ID, strOtaPackageId);
+        try {
+            OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId));
+            return checkNotNull(otaPackageService.findOtaPackageInfoById(getTenantId(), otaPackageId));
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
+
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
+    @RequestMapping(value = "/otaPackage/{otaPackageId}", method = RequestMethod.GET)
+    @ResponseBody
+    public OtaPackage getOtaPackageById(@PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException {
+        checkParameter(OTA_PACKAGE_ID, strOtaPackageId);
+        try {
+            OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId));
+            return checkOtaPackageId(otaPackageId, Operation.READ);
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
+
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
+    @RequestMapping(value = "/otaPackage", method = RequestMethod.POST)
+    @ResponseBody
+    public OtaPackageInfo saveOtaPackageInfo(@RequestBody OtaPackageInfo otaPackageInfo) throws ThingsboardException {
+        boolean created = otaPackageInfo.getId() == null;
+        try {
+            otaPackageInfo.setTenantId(getTenantId());
+            checkEntity(otaPackageInfo.getId(), otaPackageInfo, Resource.OTA_PACKAGE);
+            OtaPackageInfo savedOtaPackageInfo = otaPackageService.saveOtaPackageInfo(otaPackageInfo);
+            logEntityAction(savedOtaPackageInfo.getId(), savedOtaPackageInfo,
+                    null, created ? ActionType.ADDED : ActionType.UPDATED, null);
+            return savedOtaPackageInfo;
+        } catch (Exception e) {
+            logEntityAction(emptyId(EntityType.OTA_PACKAGE), otaPackageInfo,
+                    null, created ? ActionType.ADDED : ActionType.UPDATED, e);
+            throw handleException(e);
+        }
+    }
+
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
+    @RequestMapping(value = "/otaPackage/{otaPackageId}", method = RequestMethod.POST)
+    @ResponseBody
+    public OtaPackageInfo saveOtaPackageData(@PathVariable(OTA_PACKAGE_ID) String strOtaPackageId,
+                                         @RequestParam(required = false) String checksum,
+                                         @RequestParam(CHECKSUM_ALGORITHM) String checksumAlgorithmStr,
+                                         @RequestBody MultipartFile file) throws ThingsboardException {
+        checkParameter(OTA_PACKAGE_ID, strOtaPackageId);
+        checkParameter(CHECKSUM_ALGORITHM, checksumAlgorithmStr);
+        try {
+            OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId));
+            OtaPackageInfo info = checkOtaPackageInfoId(otaPackageId, Operation.READ);
+
+            OtaPackage otaPackage = new OtaPackage(otaPackageId);
+            otaPackage.setCreatedTime(info.getCreatedTime());
+            otaPackage.setTenantId(getTenantId());
+            otaPackage.setDeviceProfileId(info.getDeviceProfileId());
+            otaPackage.setType(info.getType());
+            otaPackage.setTitle(info.getTitle());
+            otaPackage.setVersion(info.getVersion());
+            otaPackage.setAdditionalInfo(info.getAdditionalInfo());
+
+            ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.valueOf(checksumAlgorithmStr.toUpperCase());
+
+            byte[] bytes = file.getBytes();
+            if (StringUtils.isEmpty(checksum)) {
+                checksum = otaPackageService.generateChecksum(checksumAlgorithm, ByteBuffer.wrap(bytes));
+            }
+
+            otaPackage.setChecksumAlgorithm(checksumAlgorithm);
+            otaPackage.setChecksum(checksum);
+            otaPackage.setFileName(file.getOriginalFilename());
+            otaPackage.setContentType(file.getContentType());
+            otaPackage.setData(ByteBuffer.wrap(bytes));
+            otaPackage.setDataSize((long) bytes.length);
+            OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage);
+            logEntityAction(savedOtaPackage.getId(), savedOtaPackage, null, ActionType.UPDATED, null);
+            return savedOtaPackage;
+        } catch (Exception e) {
+            logEntityAction(emptyId(EntityType.OTA_PACKAGE), null, null, ActionType.UPDATED, e, strOtaPackageId);
+            throw handleException(e);
+        }
+    }
+
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
+    @RequestMapping(value = "/otaPackages", method = RequestMethod.GET)
+    @ResponseBody
+    public PageData<OtaPackageInfo> getOtaPackages(@RequestParam int pageSize,
+                                                 @RequestParam int page,
+                                                 @RequestParam(required = false) String textSearch,
+                                                 @RequestParam(required = false) String sortProperty,
+                                                 @RequestParam(required = false) String sortOrder) throws ThingsboardException {
+        try {
+            PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder);
+            return checkNotNull(otaPackageService.findTenantOtaPackagesByTenantId(getTenantId(), pageLink));
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
+
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
+    @RequestMapping(value = "/otaPackages/{deviceProfileId}/{type}", method = RequestMethod.GET)
+    @ResponseBody
+    public PageData<OtaPackageInfo> getOtaPackages(@PathVariable("deviceProfileId") String strDeviceProfileId,
+                                                   @PathVariable("type") String strType,
+                                                   @RequestParam int pageSize,
+                                                   @RequestParam int page,
+                                                   @RequestParam(required = false) String textSearch,
+                                                   @RequestParam(required = false) String sortProperty,
+                                                   @RequestParam(required = false) String sortOrder) throws ThingsboardException {
+        checkParameter("deviceProfileId", strDeviceProfileId);
+        checkParameter("type", strType);
+        try {
+            PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder);
+            return checkNotNull(otaPackageService.findTenantOtaPackagesByTenantIdAndDeviceProfileIdAndTypeAndHasData(getTenantId(),
+                    new DeviceProfileId(toUUID(strDeviceProfileId)), OtaPackageType.valueOf(strType), pageLink));
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
+
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
+    @RequestMapping(value = "/otaPackage/{otaPackageId}", method = RequestMethod.DELETE)
+    @ResponseBody
+    public void deleteOtaPackage(@PathVariable("otaPackageId") String strOtaPackageId) throws ThingsboardException {
+        checkParameter(OTA_PACKAGE_ID, strOtaPackageId);
+        try {
+            OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId));
+            OtaPackageInfo info = checkOtaPackageInfoId(otaPackageId, Operation.DELETE);
+            otaPackageService.deleteOtaPackage(getTenantId(), otaPackageId);
+            logEntityAction(otaPackageId, info, null, ActionType.DELETED, null, strOtaPackageId);
+        } catch (Exception e) {
+            logEntityAction(emptyId(EntityType.OTA_PACKAGE), null, null, ActionType.DELETED, e, strOtaPackageId);
+            throw handleException(e);
+        }
+    }
+
+}

application/src/main/java/org/thingsboard/server/controller/QueueController.java

@@ -15,6 +15,7 @@
  */
 package org.thingsboard.server.controller;
 
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
@@ -23,17 +24,23 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.RestController;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.msg.queue.ServiceType;
+import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
+import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
 @RestController
 @TbCoreComponent
 @RequestMapping("/api")
 public class QueueController extends BaseController {
 
+    @Autowired(required = false)
+    private TbQueueRuleEngineSettings ruleEngineSettings;
+
     @PreAuthorize("hasAuthority('TENANT_ADMIN')")
     @RequestMapping(value = "/tenant/queues", params = {"serviceType"}, method = RequestMethod.GET)
     @ResponseBody
@@ -43,7 +50,12 @@ public class QueueController extends BaseController {
             ServiceType type = ServiceType.valueOf(serviceType);
             switch (type) {
                 case TB_RULE_ENGINE:
-                    return Arrays.asList("Main", "HighPriority", "SequentialByOriginator");
+                    if (ruleEngineSettings == null) {
+                        return Arrays.asList("Main", "HighPriority", "SequentialByOriginator");
+                    }
+                    return ruleEngineSettings.getQueues().stream()
+                            .map(TbRuleEngineQueueConfiguration::getName)
+                            .collect(Collectors.toList());
                 default:
                     return Collections.emptyList();
             }

application/src/main/java/org/thingsboard/server/controller/RpcController.java

@@ -29,6 +29,7 @@ import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.context.request.async.DeferredResult;
@@ -38,9 +39,13 @@ import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.RpcId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.id.UUIDBased;
-import org.thingsboard.server.common.data.rpc.RpcRequest;
+import org.thingsboard.server.common.data.page.PageData;
+import org.thingsboard.server.common.data.page.PageLink;
+import org.thingsboard.server.common.data.rpc.Rpc;
+import org.thingsboard.server.common.data.rpc.RpcStatus;
 import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody;
 import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest;
 import org.thingsboard.server.queue.util.TbCoreComponent;
@@ -94,32 +99,75 @@ public class RpcController extends BaseController {
         return handleDeviceRPCRequest(false, new DeviceId(UUID.fromString(deviceIdStr)), requestBody);
     }
 
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
+    @RequestMapping(value = "/persisted/{rpcId}", method = RequestMethod.GET)
+    @ResponseBody
+    public Rpc getPersistedRpc(@PathVariable("rpcId") String strRpc) throws ThingsboardException {
+        checkParameter("RpcId", strRpc);
+        try {
+            RpcId rpcId = new RpcId(UUID.fromString(strRpc));
+            return checkRpcId(rpcId, Operation.READ);
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
+
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
+    @RequestMapping(value = "/persisted/{deviceId}", method = RequestMethod.GET)
+    @ResponseBody
+    public PageData<Rpc> getPersistedRpcByDevice(@PathVariable("deviceId") String strDeviceId,
+                                                 @RequestParam int pageSize,
+                                                 @RequestParam int page,
+                                                 @RequestParam RpcStatus rpcStatus,
+                                                 @RequestParam(required = false) String textSearch,
+                                                 @RequestParam(required = false) String sortProperty,
+                                                 @RequestParam(required = false) String sortOrder) throws ThingsboardException {
+        checkParameter("DeviceId", strDeviceId);
+        try {
+            TenantId tenantId = getCurrentUser().getTenantId();
+            PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder);
+            DeviceId deviceId = new DeviceId(UUID.fromString(strDeviceId));
+            return checkNotNull(rpcService.findAllByDeviceIdAndStatus(tenantId, deviceId, rpcStatus, pageLink));
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
+
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
+    @RequestMapping(value = "/persisted/{rpcId}", method = RequestMethod.DELETE)
+    @ResponseBody
+    public void deleteResource(@PathVariable("rpcId") String strRpc) throws ThingsboardException {
+        checkParameter("RpcId", strRpc);
+        try {
+            rpcService.deleteRpc(getTenantId(), new RpcId(UUID.fromString(strRpc)));
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
+
     private DeferredResult<ResponseEntity> handleDeviceRPCRequest(boolean oneWay, DeviceId deviceId, String requestBody) throws ThingsboardException {
         try {
             JsonNode rpcRequestBody = jsonMapper.readTree(requestBody);
-            RpcRequest cmd = new RpcRequest(rpcRequestBody.get("method").asText(),
-                    jsonMapper.writeValueAsString(rpcRequestBody.get("params")));
-
-            if (rpcRequestBody.has("timeout")) {
-                cmd.setTimeout(rpcRequestBody.get("timeout").asLong());
-            }
+            ToDeviceRpcRequestBody body = new ToDeviceRpcRequestBody(rpcRequestBody.get("method").asText(), jsonMapper.writeValueAsString(rpcRequestBody.get("params")));
             SecurityUser currentUser = getCurrentUser();
             TenantId tenantId = currentUser.getTenantId();
             final DeferredResult<ResponseEntity> response = new DeferredResult<>();
-            long timeout = cmd.getTimeout() != null ? cmd.getTimeout() : defaultTimeout;
+            long timeout = rpcRequestBody.has("timeout") ? rpcRequestBody.get("timeout").asLong() : defaultTimeout;
             long expTime = System.currentTimeMillis() + Math.max(minTimeout, timeout);
-            ToDeviceRpcRequestBody body = new ToDeviceRpcRequestBody(cmd.getMethodName(), cmd.getRequestData());
+            UUID rpcRequestUUID = rpcRequestBody.has("requestUUID") ? UUID.fromString(rpcRequestBody.get("requestUUID").asText()) : UUID.randomUUID();
+            boolean persisted = rpcRequestBody.has("persisted") && rpcRequestBody.get("persisted").asBoolean();
             accessValidator.validate(currentUser, Operation.RPC_CALL, deviceId, new HttpValidationCallback(response, new FutureCallback<DeferredResult<ResponseEntity>>() {
                 @Override
                 public void onSuccess(@Nullable DeferredResult<ResponseEntity> result) {
-                    ToDeviceRpcRequest rpcRequest = new ToDeviceRpcRequest(UUID.randomUUID(),
+                    ToDeviceRpcRequest rpcRequest = new ToDeviceRpcRequest(rpcRequestUUID,
                             tenantId,
                             deviceId,
                             oneWay,
                             expTime,
-                            body
+                            body,
+                            persisted
                     );
-                    deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse));
+                    deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse), currentUser);
                 }
 
                 @Override

application/src/main/java/org/thingsboard/server/controller/RuleChainController.java

@@ -74,6 +74,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -86,6 +87,7 @@ public class RuleChainController extends BaseController {
     public static final String RULE_NODE_ID = "ruleNodeId";
 
     private static final ObjectMapper objectMapper = new ObjectMapper();
+    public static final int TIMEOUT = 20;
 
     @Autowired
     private InstallScripts installScripts;
@@ -244,7 +246,8 @@ public class RuleChainController extends BaseController {
             }
 
             RuleChain ruleChain = checkRuleChain(ruleChainMetaData.getRuleChainId(), Operation.WRITE);
-            RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData));
+            checkNotNull(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData) ? true : null);
+            RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaData.getRuleChainId()));
 
             if (RuleChainType.CORE.equals(ruleChain.getType())) {
                 tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED);
@@ -387,25 +390,25 @@ public class RuleChainController extends BaseController {
                 TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data);
                 switch (scriptType) {
                     case "update":
-                        output = msgToOutput(engine.executeUpdate(inMsg));
+                        output = msgToOutput(engine.executeUpdateAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS));
                         break;
                     case "generate":
-                        output = msgToOutput(engine.executeGenerate(inMsg));
+                        output = msgToOutput(engine.executeGenerateAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS));
                         break;
                     case "filter":
-                        boolean result = engine.executeFilter(inMsg);
+                        boolean result = engine.executeFilterAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS);
                         output = Boolean.toString(result);
                         break;
                     case "switch":
-                        Set<String> states = engine.executeSwitch(inMsg);
+                        Set<String> states = engine.executeSwitchAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS);
                         output = objectMapper.writeValueAsString(states);
                         break;
                     case "json":
-                        JsonNode json = engine.executeJson(inMsg);
+                        JsonNode json = engine.executeJsonAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS);
                         output = objectMapper.writeValueAsString(json);
                         break;
                     case "string":
-                        output = engine.executeToString(inMsg);
+                        output = engine.executeToStringAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS);
                         break;
                     default:
                         throw new IllegalArgumentException("Unsupported script type: " + scriptType);

application/src/main/java/org/thingsboard/server/controller/TbResourceController.java

@@ -28,15 +28,16 @@ import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.RestController;
+import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.TbResource;
 import org.thingsboard.server.common.data.TbResourceInfo;
+import org.thingsboard.server.common.data.audit.ActionType;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.id.TbResourceId;
 import org.thingsboard.server.common.data.lwm2m.LwM2mObject;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.page.PageLink;
 import org.thingsboard.server.common.data.security.Authority;
-import org.thingsboard.server.dao.resource.TbResourceService;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.security.permission.Operation;
 import org.thingsboard.server.service.security.permission.Resource;
@@ -52,12 +53,6 @@ public class TbResourceController extends BaseController {
 
     public static final String RESOURCE_ID = "resourceId";
 
-    private final TbResourceService resourceService;
-
-    public TbResourceController(TbResourceService resourceService) {
-        this.resourceService = resourceService;
-    }
-
     @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
     @RequestMapping(value = "/resource/{resourceId}/download", method = RequestMethod.GET)
     @ResponseBody
@@ -109,12 +104,18 @@ public class TbResourceController extends BaseController {
     @RequestMapping(value = "/resource", method = RequestMethod.POST)
     @ResponseBody
     public TbResource saveResource(@RequestBody TbResource resource) throws ThingsboardException {
+        boolean created = resource.getId() == null;
         try {
-                    resource.setTenantId(getTenantId());
-                    checkEntity(resource.getId(), resource, Resource.TB_RESOURCE);
-                    return addResource(resource);
-                }
-         catch (Exception e) {
+            resource.setTenantId(getTenantId());
+            checkEntity(resource.getId(), resource, Resource.TB_RESOURCE);
+            TbResource savedResource = checkNotNull(resourceService.saveResource(resource));
+            tbClusterService.onResourceChange(savedResource, null);
+            logEntityAction(savedResource.getId(), savedResource,
+                    null, created ? ActionType.ADDED : ActionType.UPDATED, null);
+            return savedResource;
+        } catch (Exception e) {
+            logEntityAction(emptyId(EntityType.TB_RESOURCE), resource,
+                    null, created ? ActionType.ADDED : ActionType.UPDATED, e);
             throw handleException(e);
         }
     }
@@ -172,21 +173,17 @@ public class TbResourceController extends BaseController {
     @RequestMapping(value = "/resource/{resourceId}", method = RequestMethod.DELETE)
     @ResponseBody
     public void deleteResource(@PathVariable("resourceId") String strResourceId) throws ThingsboardException {
-        checkParameter("resourceId", strResourceId);
+        checkParameter(RESOURCE_ID, strResourceId);
         try {
             TbResourceId resourceId = new TbResourceId(toUUID(strResourceId));
             TbResource tbResource = checkResourceId(resourceId, Operation.DELETE);
             resourceService.deleteResource(getTenantId(), resourceId);
             tbClusterService.onResourceDeleted(tbResource, null);
+            logEntityAction(resourceId, tbResource, null, ActionType.DELETED, null, strResourceId);
         } catch (Exception e) {
+            logEntityAction(emptyId(EntityType.TB_RESOURCE), null, null, ActionType.DELETED, e, strResourceId);
             throw handleException(e);
         }
     }
 
-    private TbResource addResource(TbResource resource) throws Exception {
-            checkEntity(resource.getId(), resource, Resource.TB_RESOURCE);
-            TbResource savedResource = checkNotNull(resourceService.saveResource(resource));
-            tbClusterService.onResourceChange(savedResource, null);
-            return savedResource;
-    }
 }

application/src/main/java/org/thingsboard/server/controller/TelemetryController.java

@@ -48,6 +48,7 @@ import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.TenantProfile;
 import org.thingsboard.server.common.data.audit.ActionType;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
+import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.EntityIdFactory;
@@ -449,7 +450,7 @@ public class TelemetryController extends BaseController {
                 TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
                 tenantTtl = TimeUnit.DAYS.toSeconds(((DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration()).getDefaultStorageTtlDays());
             }
-            tsSubService.saveAndNotify(tenantId, entityId, entries, tenantTtl, new FutureCallback<Void>() {
+            tsSubService.saveAndNotify(tenantId, user.getCustomerId(), entityId, entries, tenantTtl, new FutureCallback<Void>() {
                 @Override
                 public void onSuccess(@Nullable Void tmp) {
                     logTelemetryUpdated(user, entityId, entries, null);

application/src/main/java/org/thingsboard/server/controller/UserController.java

@@ -32,6 +32,7 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestController;
+import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.rule.engine.api.MailService;
 import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.User;
@@ -89,13 +90,14 @@ public class UserController extends BaseController {
         try {
             UserId userId = new UserId(toUUID(strUserId));
             User user = checkUserId(userId, Operation.READ);
-            if(!user.getAdditionalInfo().isNull()) {
-                processDashboardIdFromAdditionalInfo((ObjectNode) user.getAdditionalInfo(), DEFAULT_DASHBOARD);
-                processDashboardIdFromAdditionalInfo((ObjectNode) user.getAdditionalInfo(), HOME_DASHBOARD);
-            }
-            UserCredentials userCredentials = userService.findUserCredentialsByUserId(user.getTenantId(), user.getId());
-            if(userCredentials.isEnabled()) {
-                addUserCredentialsEnabled((ObjectNode) user.getAdditionalInfo());
+            if(user.getAdditionalInfo().isObject()) {
+                ObjectNode additionalInfo = (ObjectNode) user.getAdditionalInfo();
+                processDashboardIdFromAdditionalInfo(additionalInfo, DEFAULT_DASHBOARD);
+                processDashboardIdFromAdditionalInfo(additionalInfo, HOME_DASHBOARD);
+                UserCredentials userCredentials = userService.findUserCredentialsByUserId(user.getTenantId(), user.getId());
+                if(userCredentials.isEnabled() && !additionalInfo.has("userCredentialsEnabled")) {
+                    additionalInfo.put("userCredentialsEnabled", true);
+                }
             }
             return user;
         } catch (Exception e) {
@@ -103,14 +105,6 @@ public class UserController extends BaseController {
         }
     }
 
-    private void addUserCredentialsEnabled(ObjectNode additionalInfo) {
-        if(!additionalInfo.isNull()) {
-            if(!additionalInfo.has("userCredentialsEnabled")) {
-                additionalInfo.put("userCredentialsEnabled", true);
-            }
-        }
-    }
-
     @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
     @RequestMapping(value = "/user/tokenAccessEnabled", method = RequestMethod.GET)
     @ResponseBody

application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@@ -193,9 +193,13 @@ public class ThingsboardInstallService {
                             databaseEntitiesUpgradeService.upgradeDatabase("3.2.1");
                         case "3.2.2":
                             log.info("Upgrading ThingsBoard from version 3.2.2 to 3.3.0 ...");
+                            if (databaseTsUpgradeService != null) {
+                                databaseTsUpgradeService.upgradeDatabase("3.2.2");
+                            }
                             databaseEntitiesUpgradeService.upgradeDatabase("3.2.2");
 
                             dataUpdateService.updateData("3.2.2");
+                            systemDataLoaderService.createOAuth2Templates();
 
                             log.info("Updating system data...");
                             systemDataLoaderService.updateSystemWidgets();
@@ -232,7 +236,6 @@ public class ThingsboardInstallService {
                 systemDataLoaderService.createAdminSettings();
                 systemDataLoaderService.loadSystemWidgets();
                 systemDataLoaderService.createOAuth2Templates();
-                systemDataLoaderService.loadSystemLwm2mResources();
 //                systemDataLoaderService.loadSystemPlugins();
 //                systemDataLoaderService.loadSystemRules();
 

application/src/main/java/org/thingsboard/server/service/action/RuleEngineEntityActionService.java

@@ -0,0 +1,256 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.action;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.common.data.DataConstants;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.HasName;
+import org.thingsboard.server.common.data.HasTenantId;
+import org.thingsboard.server.common.data.User;
+import org.thingsboard.server.common.data.audit.ActionType;
+import org.thingsboard.server.common.data.id.CustomerId;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.kv.AttributeKvEntry;
+import org.thingsboard.server.common.data.kv.DataType;
+import org.thingsboard.server.common.data.kv.KvEntry;
+import org.thingsboard.server.common.data.kv.TsKvEntry;
+import org.thingsboard.server.common.msg.TbMsg;
+import org.thingsboard.server.common.msg.TbMsgDataType;
+import org.thingsboard.server.common.msg.TbMsgMetaData;
+import org.thingsboard.server.queue.util.TbCoreComponent;
+import org.thingsboard.server.service.queue.TbClusterService;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@TbCoreComponent
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class RuleEngineEntityActionService {
+    private final TbClusterService tbClusterService;
+
+    private static final ObjectMapper json = new ObjectMapper();
+
+    public void pushEntityActionToRuleEngine(EntityId entityId, HasName entity, TenantId tenantId, CustomerId customerId,
+                                             ActionType actionType, User user, Object... additionalInfo) {
+        String msgType = null;
+        switch (actionType) {
+            case ADDED:
+                msgType = DataConstants.ENTITY_CREATED;
+                break;
+            case DELETED:
+                msgType = DataConstants.ENTITY_DELETED;
+                break;
+            case UPDATED:
+                msgType = DataConstants.ENTITY_UPDATED;
+                break;
+            case ASSIGNED_TO_CUSTOMER:
+                msgType = DataConstants.ENTITY_ASSIGNED;
+                break;
+            case UNASSIGNED_FROM_CUSTOMER:
+                msgType = DataConstants.ENTITY_UNASSIGNED;
+                break;
+            case ATTRIBUTES_UPDATED:
+                msgType = DataConstants.ATTRIBUTES_UPDATED;
+                break;
+            case ATTRIBUTES_DELETED:
+                msgType = DataConstants.ATTRIBUTES_DELETED;
+                break;
+            case ALARM_ACK:
+                msgType = DataConstants.ALARM_ACK;
+                break;
+            case ALARM_CLEAR:
+                msgType = DataConstants.ALARM_CLEAR;
+                break;
+            case ALARM_DELETE:
+                msgType = DataConstants.ALARM_DELETE;
+                break;
+            case ASSIGNED_FROM_TENANT:
+                msgType = DataConstants.ENTITY_ASSIGNED_FROM_TENANT;
+                break;
+            case ASSIGNED_TO_TENANT:
+                msgType = DataConstants.ENTITY_ASSIGNED_TO_TENANT;
+                break;
+            case PROVISION_SUCCESS:
+                msgType = DataConstants.PROVISION_SUCCESS;
+                break;
+            case PROVISION_FAILURE:
+                msgType = DataConstants.PROVISION_FAILURE;
+                break;
+            case TIMESERIES_UPDATED:
+                msgType = DataConstants.TIMESERIES_UPDATED;
+                break;
+            case TIMESERIES_DELETED:
+                msgType = DataConstants.TIMESERIES_DELETED;
+                break;
+            case ASSIGNED_TO_EDGE:
+                msgType = DataConstants.ENTITY_ASSIGNED_TO_EDGE;
+                break;
+            case UNASSIGNED_FROM_EDGE:
+                msgType = DataConstants.ENTITY_UNASSIGNED_FROM_EDGE;
+                break;
+        }
+        if (!StringUtils.isEmpty(msgType)) {
+            try {
+                TbMsgMetaData metaData = new TbMsgMetaData();
+                if (user != null) {
+                    metaData.putValue("userId", user.getId().toString());
+                    metaData.putValue("userName", user.getName());
+                }
+                if (customerId != null && !customerId.isNullUid()) {
+                    metaData.putValue("customerId", customerId.toString());
+                }
+                if (actionType == ActionType.ASSIGNED_TO_CUSTOMER) {
+                    String strCustomerId = extractParameter(String.class, 1, additionalInfo);
+                    String strCustomerName = extractParameter(String.class, 2, additionalInfo);
+                    metaData.putValue("assignedCustomerId", strCustomerId);
+                    metaData.putValue("assignedCustomerName", strCustomerName);
+                } else if (actionType == ActionType.UNASSIGNED_FROM_CUSTOMER) {
+                    String strCustomerId = extractParameter(String.class, 1, additionalInfo);
+                    String strCustomerName = extractParameter(String.class, 2, additionalInfo);
+                    metaData.putValue("unassignedCustomerId", strCustomerId);
+                    metaData.putValue("unassignedCustomerName", strCustomerName);
+                } else if (actionType == ActionType.ASSIGNED_FROM_TENANT) {
+                    String strTenantId = extractParameter(String.class, 0, additionalInfo);
+                    String strTenantName = extractParameter(String.class, 1, additionalInfo);
+                    metaData.putValue("assignedFromTenantId", strTenantId);
+                    metaData.putValue("assignedFromTenantName", strTenantName);
+                } else if (actionType == ActionType.ASSIGNED_TO_TENANT) {
+                    String strTenantId = extractParameter(String.class, 0, additionalInfo);
+                    String strTenantName = extractParameter(String.class, 1, additionalInfo);
+                    metaData.putValue("assignedToTenantId", strTenantId);
+                    metaData.putValue("assignedToTenantName", strTenantName);
+                } else if (actionType == ActionType.ASSIGNED_TO_EDGE) {
+                    String strEdgeId = extractParameter(String.class, 1, additionalInfo);
+                    String strEdgeName = extractParameter(String.class, 2, additionalInfo);
+                    metaData.putValue("assignedEdgeId", strEdgeId);
+                    metaData.putValue("assignedEdgeName", strEdgeName);
+                } else if (actionType == ActionType.UNASSIGNED_FROM_EDGE) {
+                    String strEdgeId = extractParameter(String.class, 1, additionalInfo);
+                    String strEdgeName = extractParameter(String.class, 2, additionalInfo);
+                    metaData.putValue("unassignedEdgeId", strEdgeId);
+                    metaData.putValue("unassignedEdgeName", strEdgeName);
+                }
+                ObjectNode entityNode;
+                if (entity != null) {
+                    entityNode = json.valueToTree(entity);
+                    if (entityId.getEntityType() == EntityType.DASHBOARD) {
+                        entityNode.put("configuration", "");
+                    }
+                } else {
+                    entityNode = json.createObjectNode();
+                    if (actionType == ActionType.ATTRIBUTES_UPDATED) {
+                        String scope = extractParameter(String.class, 0, additionalInfo);
+                        @SuppressWarnings("unchecked")
+                        List<AttributeKvEntry> attributes = extractParameter(List.class, 1, additionalInfo);
+                        metaData.putValue(DataConstants.SCOPE, scope);
+                        if (attributes != null) {
+                            for (AttributeKvEntry attr : attributes) {
+                                addKvEntry(entityNode, attr);
+                            }
+                        }
+                    } else if (actionType == ActionType.ATTRIBUTES_DELETED) {
+                        String scope = extractParameter(String.class, 0, additionalInfo);
+                        @SuppressWarnings("unchecked")
+                        List<String> keys = extractParameter(List.class, 1, additionalInfo);
+                        metaData.putValue(DataConstants.SCOPE, scope);
+                        ArrayNode attrsArrayNode = entityNode.putArray("attributes");
+                        if (keys != null) {
+                            keys.forEach(attrsArrayNode::add);
+                        }
+                    } else if (actionType == ActionType.TIMESERIES_UPDATED) {
+                        @SuppressWarnings("unchecked")
+                        List<TsKvEntry> timeseries = extractParameter(List.class, 0, additionalInfo);
+                        addTimeseries(entityNode, timeseries);
+                    } else if (actionType == ActionType.TIMESERIES_DELETED) {
+                        @SuppressWarnings("unchecked")
+                        List<String> keys = extractParameter(List.class, 0, additionalInfo);
+                        if (keys != null) {
+                            ArrayNode timeseriesArrayNode = entityNode.putArray("timeseries");
+                            keys.forEach(timeseriesArrayNode::add);
+                        }
+                        entityNode.put("startTs", extractParameter(Long.class, 1, additionalInfo));
+                        entityNode.put("endTs", extractParameter(Long.class, 2, additionalInfo));
+                    }
+                }
+                TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, customerId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode));
+                if (tenantId.isNullUid()) {
+                    if (entity instanceof HasTenantId) {
+                        tenantId = ((HasTenantId) entity).getTenantId();
+                    }
+                }
+                tbClusterService.pushMsgToRuleEngine(tenantId, entityId, tbMsg, null);
+            } catch (Exception e) {
+                log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e);
+            }
+        }
+    }
+
+
+    private <T> T extractParameter(Class<T> clazz, int index, Object... additionalInfo) {
+        T result = null;
+        if (additionalInfo != null && additionalInfo.length > index) {
+            Object paramObject = additionalInfo[index];
+            if (clazz.isInstance(paramObject)) {
+                result = clazz.cast(paramObject);
+            }
+        }
+        return result;
+    }
+
+    private void addTimeseries(ObjectNode entityNode, List<TsKvEntry> timeseries) throws Exception {
+        if (timeseries != null && !timeseries.isEmpty()) {
+            ArrayNode result = entityNode.putArray("timeseries");
+            Map<Long, List<TsKvEntry>> groupedTelemetry = timeseries.stream()
+                    .collect(Collectors.groupingBy(TsKvEntry::getTs));
+            for (Map.Entry<Long, List<TsKvEntry>> entry : groupedTelemetry.entrySet()) {
+                ObjectNode element = json.createObjectNode();
+                element.put("ts", entry.getKey());
+                ObjectNode values = element.putObject("values");
+                for (TsKvEntry tsKvEntry : entry.getValue()) {
+                    addKvEntry(values, tsKvEntry);
+                }
+                result.add(element);
+            }
+        }
+    }
+
+    private void addKvEntry(ObjectNode entityNode, KvEntry kvEntry) throws Exception {
+        if (kvEntry.getDataType() == DataType.BOOLEAN) {
+            kvEntry.getBooleanValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
+        } else if (kvEntry.getDataType() == DataType.DOUBLE) {
+            kvEntry.getDoubleValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
+        } else if (kvEntry.getDataType() == DataType.LONG) {
+            kvEntry.getLongValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
+        } else if (kvEntry.getDataType() == DataType.JSON) {
+            if (kvEntry.getJsonValue().isPresent()) {
+                entityNode.set(kvEntry.getKey(), json.readTree(kvEntry.getJsonValue().get()));
+            }
+        } else {
+            entityNode.put(kvEntry.getKey(), kvEntry.getValueAsString());
+        }
+    }
+}

application/src/main/java/org/thingsboard/server/service/apiusage/BaseApiUsageState.java

@@ -0,0 +1,153 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.apiusage;
+
+import lombok.Getter;
+import org.springframework.data.util.Pair;
+import org.thingsboard.server.common.data.ApiFeature;
+import org.thingsboard.server.common.data.ApiUsageRecordKey;
+import org.thingsboard.server.common.data.ApiUsageState;
+import org.thingsboard.server.common.data.ApiUsageStateValue;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.msg.tools.SchedulerUtils;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+public abstract class BaseApiUsageState {
+    private final Map<ApiUsageRecordKey, Long> currentCycleValues = new ConcurrentHashMap<>();
+    private final Map<ApiUsageRecordKey, Long> currentHourValues = new ConcurrentHashMap<>();
+
+    @Getter
+    private final ApiUsageState apiUsageState;
+    @Getter
+    private volatile long currentCycleTs;
+    @Getter
+    private volatile long nextCycleTs;
+    @Getter
+    private volatile long currentHourTs;
+
+    public BaseApiUsageState(ApiUsageState apiUsageState) {
+        this.apiUsageState = apiUsageState;
+        this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth();
+        this.nextCycleTs = SchedulerUtils.getStartOfNextMonth();
+        this.currentHourTs = SchedulerUtils.getStartOfCurrentHour();
+    }
+
+    public void put(ApiUsageRecordKey key, Long value) {
+        currentCycleValues.put(key, value);
+    }
+
+    public void putHourly(ApiUsageRecordKey key, Long value) {
+        currentHourValues.put(key, value);
+    }
+
+    public long add(ApiUsageRecordKey key, long value) {
+        long result = currentCycleValues.getOrDefault(key, 0L) + value;
+        currentCycleValues.put(key, result);
+        return result;
+    }
+
+    public long get(ApiUsageRecordKey key) {
+        return currentCycleValues.getOrDefault(key, 0L);
+    }
+
+    public long addToHourly(ApiUsageRecordKey key, long value) {
+        long result = currentHourValues.getOrDefault(key, 0L) + value;
+        currentHourValues.put(key, result);
+        return result;
+    }
+
+    public void setHour(long currentHourTs) {
+        this.currentHourTs = currentHourTs;
+        for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
+            currentHourValues.put(key, 0L);
+        }
+    }
+
+    public void setCycles(long currentCycleTs, long nextCycleTs) {
+        this.currentCycleTs = currentCycleTs;
+        this.nextCycleTs = nextCycleTs;
+        for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
+            currentCycleValues.put(key, 0L);
+        }
+    }
+
+    public ApiUsageStateValue getFeatureValue(ApiFeature feature) {
+        switch (feature) {
+            case TRANSPORT:
+                return apiUsageState.getTransportState();
+            case RE:
+                return apiUsageState.getReExecState();
+            case DB:
+                return apiUsageState.getDbStorageState();
+            case JS:
+                return apiUsageState.getJsExecState();
+            case EMAIL:
+                return apiUsageState.getEmailExecState();
+            case SMS:
+                return apiUsageState.getSmsExecState();
+            case ALARM:
+                return apiUsageState.getAlarmExecState();
+            default:
+                return ApiUsageStateValue.ENABLED;
+        }
+    }
+
+    public boolean setFeatureValue(ApiFeature feature, ApiUsageStateValue value) {
+        ApiUsageStateValue currentValue = getFeatureValue(feature);
+        switch (feature) {
+            case TRANSPORT:
+                apiUsageState.setTransportState(value);
+                break;
+            case RE:
+                apiUsageState.setReExecState(value);
+                break;
+            case DB:
+                apiUsageState.setDbStorageState(value);
+                break;
+            case JS:
+                apiUsageState.setJsExecState(value);
+                break;
+            case EMAIL:
+                apiUsageState.setEmailExecState(value);
+                break;
+            case SMS:
+                apiUsageState.setSmsExecState(value);
+                break;
+            case ALARM:
+                apiUsageState.setAlarmExecState(value);
+                break;
+        }
+        return !currentValue.equals(value);
+    }
+
+    public abstract EntityType getEntityType();
+
+    public TenantId getTenantId() {
+        return getApiUsageState().getTenantId();
+    }
+
+    public EntityId getEntityId() {
+        return getApiUsageState().getEntityId();
+    }
+}

application/src/main/java/org/thingsboard/server/service/apiusage/CustomerApiUsageState.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.apiusage;
+
+import org.thingsboard.server.common.data.ApiUsageState;
+import org.thingsboard.server.common.data.EntityType;
+
+public class CustomerApiUsageState extends BaseApiUsageState {
+    public CustomerApiUsageState(ApiUsageState apiUsageState) {
+        super(apiUsageState);
+    }
+
+    @Override
+    public EntityType getEntityType() {
+        return EntityType.CUSTOMER;
+    }
+}

application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java

@@ -23,16 +23,20 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.ThingsBoardThreadFactory;
 import org.thingsboard.rule.engine.api.MailService;
 import org.thingsboard.server.common.data.ApiFeature;
 import org.thingsboard.server.common.data.ApiUsageRecordKey;
 import org.thingsboard.server.common.data.ApiUsageState;
 import org.thingsboard.server.common.data.ApiUsageStateMailMessage;
 import org.thingsboard.server.common.data.ApiUsageStateValue;
+import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.Tenant;
 import org.thingsboard.server.common.data.TenantProfile;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.id.ApiUsageStateId;
+import org.thingsboard.server.common.data.id.CustomerId;
+import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.id.TenantProfileId;
 import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
@@ -45,6 +49,7 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
 import org.thingsboard.server.common.msg.queue.ServiceType;
 import org.thingsboard.server.common.msg.queue.TbCallback;
 import org.thingsboard.server.common.msg.tools.SchedulerUtils;
+import org.thingsboard.server.dao.customer.CustomerService;
 import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
 import org.thingsboard.server.dao.tenant.TenantService;
 import org.thingsboard.server.dao.timeseries.TimeseriesService;
@@ -52,7 +57,7 @@ import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
 import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto;
 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
-import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
+import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
 import org.thingsboard.server.queue.discovery.PartitionService;
 import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
 import org.thingsboard.server.queue.scheduler.SchedulerComponent;
@@ -63,6 +68,7 @@ import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -95,6 +101,7 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
     private final TbClusterService clusterService;
     private final PartitionService partitionService;
     private final TenantService tenantService;
+    private final CustomerService customerService;
     private final TimeseriesService tsService;
     private final ApiUsageStateService apiUsageStateService;
     private final SchedulerComponent scheduler;
@@ -105,10 +112,12 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
     @Autowired
     private InternalTelemetryService tsWsService;
 
-    // Tenants that should be processed on this server
-    private final Map<TenantId, TenantApiUsageState> myTenantStates = new ConcurrentHashMap<>();
-    // Tenants that should be processed on other servers
-    private final Map<TenantId, ApiUsageState> otherTenantStates = new ConcurrentHashMap<>();
+    // Entities that should be processed on this server
+    private final Map<EntityId, BaseApiUsageState> myUsageStates = new ConcurrentHashMap<>();
+    // Entities that should be processed on other servers
+    private final Map<EntityId, ApiUsageState> otherUsageStates = new ConcurrentHashMap<>();
+
+    private final Set<EntityId> deletedEntities = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
     @Value("${usage.stats.report.enabled:true}")
     private boolean enabled;
@@ -123,19 +132,22 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
     public DefaultTbApiUsageStateService(TbClusterService clusterService,
                                          PartitionService partitionService,
                                          TenantService tenantService,
+                                         CustomerService customerService,
                                          TimeseriesService tsService,
                                          ApiUsageStateService apiUsageStateService,
                                          SchedulerComponent scheduler,
-                                         TbTenantProfileCache tenantProfileCache, MailService mailService) {
+                                         TbTenantProfileCache tenantProfileCache,
+                                         MailService mailService) {
         this.clusterService = clusterService;
         this.partitionService = partitionService;
         this.tenantService = tenantService;
+        this.customerService = customerService;
         this.tsService = tsService;
         this.apiUsageStateService = apiUsageStateService;
         this.scheduler = scheduler;
         this.tenantProfileCache = tenantProfileCache;
         this.mailService = mailService;
-        this.mailExecutor = Executors.newSingleThreadExecutor();
+        this.mailExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("api-usage-svc-mail"));
     }
 
     @PostConstruct
@@ -150,74 +162,92 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
     @Override
     public void process(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
         ToUsageStatsServiceMsg statsMsg = msg.getValue();
-        TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB()));
 
-        if (tenantProfileCache.get(tenantId) == null) {
-            return;
+        TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB()));
+        EntityId entityId;
+        if (statsMsg.getCustomerIdMSB() != 0 && statsMsg.getCustomerIdLSB() != 0) {
+            entityId = new CustomerId(new UUID(statsMsg.getCustomerIdMSB(), statsMsg.getCustomerIdLSB()));
+        } else {
+            entityId = tenantId;
         }
 
-        TenantApiUsageState tenantState;
+        processEntityUsageStats(tenantId, entityId, statsMsg.getValuesList());
+        callback.onSuccess();
+    }
+
+    private void processEntityUsageStats(TenantId tenantId, EntityId entityId, List<UsageStatsKVProto> values) {
+        if (deletedEntities.contains(entityId)) return;
+
+        BaseApiUsageState usageState;
         List<TsKvEntry> updatedEntries;
         Map<ApiFeature, ApiUsageStateValue> result;
+
         updateLock.lock();
         try {
-            tenantState = getOrFetchState(tenantId);
-            long ts = tenantState.getCurrentCycleTs();
-            long hourTs = tenantState.getCurrentHourTs();
+            usageState = getOrFetchState(tenantId, entityId);
+            long ts = usageState.getCurrentCycleTs();
+            long hourTs = usageState.getCurrentHourTs();
             long newHourTs = SchedulerUtils.getStartOfCurrentHour();
             if (newHourTs != hourTs) {
-                tenantState.setHour(newHourTs);
+                usageState.setHour(newHourTs);
             }
             updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length);
             Set<ApiFeature> apiFeatures = new HashSet<>();
-            for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) {
+            for (UsageStatsKVProto kvProto : values) {
                 ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey());
-                long newValue = tenantState.add(recordKey, kvProto.getValue());
+                long newValue = usageState.add(recordKey, kvProto.getValue());
                 updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.getApiCountKey(), newValue)));
-                long newHourlyValue = tenantState.addToHourly(recordKey, kvProto.getValue());
+                long newHourlyValue = usageState.addToHourly(recordKey, kvProto.getValue());
                 updatedEntries.add(new BasicTsKvEntry(newHourTs, new LongDataEntry(recordKey.getApiCountKey() + HOURLY, newHourlyValue)));
                 apiFeatures.add(recordKey.getApiFeature());
             }
-            result = tenantState.checkStateUpdatedDueToThreshold(apiFeatures);
+            if (usageState.getEntityType() == EntityType.TENANT && !usageState.getEntityId().equals(TenantId.SYS_TENANT_ID)) {
+                result = ((TenantApiUsageState) usageState).checkStateUpdatedDueToThreshold(apiFeatures);
+            } else {
+                result = Collections.emptyMap();
+            }
         } finally {
             updateLock.unlock();
         }
-        tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK);
+        tsWsService.saveAndNotifyInternal(tenantId, usageState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK);
         if (!result.isEmpty()) {
-            persistAndNotify(tenantState, result);
+            persistAndNotify(usageState, result);
         }
-        callback.onSuccess();
     }
 
     @Override
     protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
         if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) {
-            myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());
-            otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());
+            myUsageStates.entrySet().removeIf(entry -> {
+                return !partitionService.resolve(ServiceType.TB_CORE, entry.getValue().getTenantId(), entry.getKey()).isMyPartition();
+            });
+            otherUsageStates.entrySet().removeIf(entry -> {
+                return partitionService.resolve(ServiceType.TB_CORE, entry.getValue().getTenantId(), entry.getKey()).isMyPartition();
+            });
             initStatesFromDataBase();
         }
     }
 
     @Override
     public ApiUsageState getApiUsageState(TenantId tenantId) {
-        TenantApiUsageState tenantState = myTenantStates.get(tenantId);
+        TenantApiUsageState tenantState = (TenantApiUsageState) myUsageStates.get(tenantId);
         if (tenantState != null) {
             return tenantState.getApiUsageState();
         } else {
-            ApiUsageState state = otherTenantStates.get(tenantId);
+            ApiUsageState state = otherUsageStates.get(tenantId);
             if (state != null) {
                 return state;
             } else {
                 if (partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) {
-                    return getOrFetchState(tenantId).getApiUsageState();
+                    return getOrFetchState(tenantId, tenantId).getApiUsageState();
                 } else {
                     updateLock.lock();
                     try {
-                        state = otherTenantStates.get(tenantId);
+                        state = otherUsageStates.get(tenantId);
                         if (state == null) {
                             state = apiUsageStateService.findTenantApiUsageState(tenantId);
                             if (state != null) {
-                                otherTenantStates.put(tenantId, state);
+                                otherUsageStates.put(tenantId, state);
                             }
                         }
                     } finally {
@@ -231,7 +261,7 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
 
     @Override
     public void onApiUsageStateUpdate(TenantId tenantId) {
-        otherTenantStates.remove(tenantId);
+        otherUsageStates.remove(tenantId);
     }
 
     @Override
@@ -240,11 +270,14 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
         TenantProfile tenantProfile = tenantProfileCache.get(tenantProfileId);
         updateLock.lock();
         try {
-            myTenantStates.values().forEach(state -> {
-                if (tenantProfile.getId().equals(state.getTenantProfileId())) {
-                    updateTenantState(state, tenantProfile);
-                }
-            });
+            myUsageStates.values().stream()
+                    .filter(state -> state.getEntityType() == EntityType.TENANT)
+                    .map(state -> (TenantApiUsageState) state)
+                    .forEach(state -> {
+                        if (tenantProfile.getId().equals(state.getTenantProfileId())) {
+                            updateTenantState(state, tenantProfile);
+                        }
+                    });
         } finally {
             updateLock.unlock();
         }
@@ -256,7 +289,7 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
         TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
         updateLock.lock();
         try {
-            TenantApiUsageState state = myTenantStates.get(tenantId);
+            TenantApiUsageState state = (TenantApiUsageState) myUsageStates.get(tenantId);
             if (state != null && !state.getTenantProfileId().equals(tenantProfile.getId())) {
                 updateTenantState(state, tenantProfile);
             }
@@ -293,29 +326,42 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
         }
     }
 
-    private void persistAndNotify(TenantApiUsageState state, Map<ApiFeature, ApiUsageStateValue> result) {
-        log.info("[{}] Detected update of the API state: {}", state.getTenantId(), result);
+    public void onTenantDelete(TenantId tenantId) {
+        deletedEntities.add(tenantId);
+        myUsageStates.remove(tenantId);
+        otherUsageStates.remove(tenantId);
+    }
+
+    @Override
+    public void onCustomerDelete(CustomerId customerId) {
+        deletedEntities.add(customerId);
+        myUsageStates.remove(customerId);
+    }
+
+    private void persistAndNotify(BaseApiUsageState state, Map<ApiFeature, ApiUsageStateValue> result) {
+        log.info("[{}] Detected update of the API state for {}: {}", state.getEntityId(), state.getEntityType(), result);
         apiUsageStateService.update(state.getApiUsageState());
         clusterService.onApiStateChange(state.getApiUsageState(), null);
         long ts = System.currentTimeMillis();
         List<TsKvEntry> stateTelemetry = new ArrayList<>();
-        result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(apiFeature.getApiStateKey(), aState.name())))));
+        result.forEach((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(apiFeature.getApiStateKey(), aState.name()))));
         tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, VOID_CALLBACK);
 
-        String email = tenantService.findTenantById(state.getTenantId()).getEmail();
-
-        if (StringUtils.isNotEmpty(email)) {
-            result.forEach((apiFeature, stateValue) -> {
-                mailExecutor.submit(() -> {
-                    try {
-                        mailService.sendApiFeatureStateEmail(apiFeature, stateValue, email, createStateMailMessage(state, apiFeature, stateValue));
-                    } catch (ThingsboardException e) {
-                        log.warn("[{}] Can't send update of the API state to tenant with provided email [{}]", state.getTenantId(), email, e);
-                    }
+        if (state.getEntityType() == EntityType.TENANT && !state.getEntityId().equals(TenantId.SYS_TENANT_ID)) {
+            String email = tenantService.findTenantById(state.getTenantId()).getEmail();
+            if (StringUtils.isNotEmpty(email)) {
+                result.forEach((apiFeature, stateValue) -> {
+                    mailExecutor.submit(() -> {
+                        try {
+                            mailService.sendApiFeatureStateEmail(apiFeature, stateValue, email, createStateMailMessage((TenantApiUsageState) state, apiFeature, stateValue));
+                        } catch (ThingsboardException e) {
+                            log.warn("[{}] Can't send update of the API state to tenant with provided email [{}]", state.getTenantId(), email, e);
+                        }
+                    });
                 });
-            });
-        } else {
-            log.warn("[{}] Can't send update of the API state to tenant with empty email!", state.getTenantId());
+            } else {
+                log.warn("[{}] Can't send update of the API state to tenant with empty email!", state.getTenantId());
+            }
         }
     }
 
@@ -350,12 +396,14 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
         updateLock.lock();
         try {
             long now = System.currentTimeMillis();
-            myTenantStates.values().forEach(state -> {
+            myUsageStates.values().forEach(state -> {
                 if ((state.getNextCycleTs() < now) && (now - state.getNextCycleTs() < TimeUnit.HOURS.toMillis(1))) {
-                    TenantId tenantId = state.getTenantId();
                     state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextNextMonth());
                     saveNewCounts(state, Arrays.asList(ApiUsageRecordKey.values()));
-                    updateTenantState(state, tenantProfileCache.get(tenantId));
+                    if (state.getEntityType() == EntityType.TENANT && !state.getEntityId().equals(TenantId.SYS_TENANT_ID)) {
+                        TenantId tenantId = state.getTenantId();
+                        updateTenantState((TenantApiUsageState) state, tenantProfileCache.get(tenantId));
+                    }
                 }
             });
         } finally {
@@ -363,7 +411,7 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
         }
     }
 
-    private void saveNewCounts(TenantApiUsageState state, List<ApiUsageRecordKey> keys) {
+    private void saveNewCounts(BaseApiUsageState state, List<ApiUsageRecordKey> keys) {
         List<TsKvEntry> counts = keys.stream()
                 .map(key -> new BasicTsKvEntry(state.getCurrentCycleTs(), new LongDataEntry(key.getApiCountKey(), 0L)))
                 .collect(Collectors.toList());
@@ -371,52 +419,66 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
         tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), counts, VOID_CALLBACK);
     }
 
-    private TenantApiUsageState getOrFetchState(TenantId tenantId) {
-        TenantApiUsageState tenantState = myTenantStates.get(tenantId);
-        if (tenantState == null) {
-            ApiUsageState dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId);
-            if (dbStateEntity == null) {
-                try {
-                    dbStateEntity = apiUsageStateService.createDefaultApiUsageState(tenantId);
-                } catch (Exception e) {
-                    dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId);
-                }
-            }
-            TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
-            tenantState = new TenantApiUsageState(tenantProfile, dbStateEntity);
-            List<ApiUsageRecordKey> newCounts = new ArrayList<>();
+    private BaseApiUsageState getOrFetchState(TenantId tenantId, EntityId entityId) {
+        if (entityId == null || entityId.isNullUid()) {
+            entityId = tenantId;
+        }
+        BaseApiUsageState state = myUsageStates.get(entityId);
+        if (state != null) {
+            return state;
+        }
+
+        ApiUsageState storedState = apiUsageStateService.findApiUsageStateByEntityId(entityId);
+        if (storedState == null) {
             try {
-                List<TsKvEntry> dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getId()).get();
-                for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
-                    boolean cycleEntryFound = false;
-                    boolean hourlyEntryFound = false;
-                    for (TsKvEntry tsKvEntry : dbValues) {
-                        if (tsKvEntry.getKey().equals(key.getApiCountKey())) {
-                            cycleEntryFound = true;
-
-                            boolean oldCount = tsKvEntry.getTs() == tenantState.getCurrentCycleTs();
-                            tenantState.put(key, oldCount ? tsKvEntry.getLongValue().get() : 0L);
-
-                            if (!oldCount) {
-                                newCounts.add(key);
-                            }
-                        } else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) {
-                            hourlyEntryFound = true;
-                            tenantState.putHourly(key, tsKvEntry.getTs() == tenantState.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L);
-                        }
-                        if (cycleEntryFound && hourlyEntryFound) {
-                            break;
+                storedState = apiUsageStateService.createDefaultApiUsageState(tenantId, entityId);
+            } catch (Exception e) {
+                storedState = apiUsageStateService.findApiUsageStateByEntityId(entityId);
+            }
+        }
+        if (entityId.getEntityType() == EntityType.TENANT) {
+            if (!entityId.equals(TenantId.SYS_TENANT_ID)) {
+                state = new TenantApiUsageState(tenantProfileCache.get((TenantId) entityId), storedState);
+            } else {
+                state = new TenantApiUsageState(storedState);
+            }
+        } else {
+            state = new CustomerApiUsageState(storedState);
+        }
+
+        List<ApiUsageRecordKey> newCounts = new ArrayList<>();
+        try {
+            List<TsKvEntry> dbValues = tsService.findAllLatest(tenantId, storedState.getId()).get();
+            for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
+                boolean cycleEntryFound = false;
+                boolean hourlyEntryFound = false;
+                for (TsKvEntry tsKvEntry : dbValues) {
+                    if (tsKvEntry.getKey().equals(key.getApiCountKey())) {
+                        cycleEntryFound = true;
+
+                        boolean oldCount = tsKvEntry.getTs() == state.getCurrentCycleTs();
+                        state.put(key, oldCount ? tsKvEntry.getLongValue().get() : 0L);
+
+                        if (!oldCount) {
+                            newCounts.add(key);
                         }
+                    } else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) {
+                        hourlyEntryFound = true;
+                        state.putHourly(key, tsKvEntry.getTs() == state.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L);
+                    }
+                    if (cycleEntryFound && hourlyEntryFound) {
+                        break;
                     }
                 }
-                log.debug("[{}] Initialized state: {}", tenantId, dbStateEntity);
-                myTenantStates.put(tenantId, tenantState);
-                saveNewCounts(tenantState, newCounts);
-            } catch (InterruptedException | ExecutionException e) {
-                log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e);
             }
+            log.debug("[{}] Initialized state: {}", entityId, storedState);
+            myUsageStates.put(entityId, state);
+            saveNewCounts(state, newCounts);
+        } catch (InterruptedException | ExecutionException e) {
+            log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e);
         }
-        return tenantState;
+
+        return state;
     }
 
     private void initStatesFromDataBase() {
@@ -429,11 +491,11 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
                     PageDataIterable<Tenant> tenantIterator = new PageDataIterable<>(tenantService::findTenants, 1024);
                     List<Future<?>> futures = new ArrayList<>();
                     for (Tenant tenant : tenantIterator) {
-                        if (!myTenantStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) {
+                        if (!myUsageStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) {
                             log.debug("[{}] Initializing tenant state.", tenant.getId());
                             futures.add(tmpInitExecutor.submit(() -> {
                                 try {
-                                    updateTenantState(getOrFetchState(tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId()));
+                                    updateTenantState((TenantApiUsageState) getOrFetchState(tenant.getId(), tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId()));
                                     log.debug("[{}] Initialized tenant state.", tenant.getId());
                                 } catch (Exception e) {
                                     log.warn("[{}] Failed to initialize tenant API state", tenant.getId(), e);

application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java

@@ -17,12 +17,13 @@ package org.thingsboard.server.service.apiusage;
 
 import org.springframework.context.ApplicationListener;
 import org.thingsboard.server.common.data.ApiUsageState;
+import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.id.TenantProfileId;
 import org.thingsboard.server.common.msg.queue.TbCallback;
 import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
-import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
+import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
 
 public interface TbApiUsageStateService extends ApplicationListener<PartitionChangeEvent> {
 
@@ -34,5 +35,9 @@ public interface TbApiUsageStateService extends ApplicationListener<PartitionCha
 
     void onTenantUpdate(TenantId tenantId);
 
+    void onTenantDelete(TenantId tenantId);
+
+    void onCustomerDelete(CustomerId customerId);
+
     void onApiUsageStateUpdate(TenantId tenantId);
 }

application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java

@@ -22,85 +22,33 @@ import org.thingsboard.server.common.data.ApiFeature;
 import org.thingsboard.server.common.data.ApiUsageRecordKey;
 import org.thingsboard.server.common.data.ApiUsageState;
 import org.thingsboard.server.common.data.ApiUsageStateValue;
+import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.TenantProfile;
-import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.id.TenantProfileId;
 import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
-import org.thingsboard.server.common.msg.tools.SchedulerUtils;
 
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class TenantApiUsageState {
-
-    private final Map<ApiUsageRecordKey, Long> currentCycleValues = new ConcurrentHashMap<>();
-    private final Map<ApiUsageRecordKey, Long> currentHourValues = new ConcurrentHashMap<>();
 
+public class TenantApiUsageState extends BaseApiUsageState {
     @Getter
     @Setter
     private TenantProfileId tenantProfileId;
     @Getter
     @Setter
     private TenantProfileData tenantProfileData;
-    @Getter
-    private final ApiUsageState apiUsageState;
-    @Getter
-    private volatile long currentCycleTs;
-    @Getter
-    private volatile long nextCycleTs;
-    @Getter
-    private volatile long currentHourTs;
 
     public TenantApiUsageState(TenantProfile tenantProfile, ApiUsageState apiUsageState) {
+        super(apiUsageState);
         this.tenantProfileId = tenantProfile.getId();
         this.tenantProfileData = tenantProfile.getProfileData();
-        this.apiUsageState = apiUsageState;
-        this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth();
-        this.nextCycleTs = SchedulerUtils.getStartOfNextMonth();
-        this.currentHourTs = SchedulerUtils.getStartOfCurrentHour();
-    }
-
-    public void put(ApiUsageRecordKey key, Long value) {
-        currentCycleValues.put(key, value);
     }
 
-    public void putHourly(ApiUsageRecordKey key, Long value) {
-        currentHourValues.put(key, value);
-    }
-
-    public long add(ApiUsageRecordKey key, long value) {
-        long result = currentCycleValues.getOrDefault(key, 0L) + value;
-        currentCycleValues.put(key, result);
-        return result;
-    }
-
-    public long get(ApiUsageRecordKey key) {
-        return currentCycleValues.getOrDefault(key, 0L);
-    }
-
-    public long addToHourly(ApiUsageRecordKey key, long value) {
-        long result = currentHourValues.getOrDefault(key, 0L) + value;
-        currentHourValues.put(key, result);
-        return result;
-    }
-
-    public void setHour(long currentHourTs) {
-        this.currentHourTs = currentHourTs;
-        for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
-            currentHourValues.put(key, 0L);
-        }
-    }
-
-    public void setCycles(long currentCycleTs, long nextCycleTs) {
-        this.currentCycleTs = currentCycleTs;
-        this.nextCycleTs = nextCycleTs;
-        for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
-            currentCycleValues.put(key, 0L);
-        }
+    public TenantApiUsageState(ApiUsageState apiUsageState) {
+        super(apiUsageState);
     }
 
     public long getProfileThreshold(ApiUsageRecordKey key) {
@@ -111,53 +59,25 @@ public class TenantApiUsageState {
         return tenantProfileData.getConfiguration().getWarnThreshold(key);
     }
 
-    public TenantId getTenantId() {
-        return apiUsageState.getTenantId();
-    }
-
-    public ApiUsageStateValue getFeatureValue(ApiFeature feature) {
-        switch (feature) {
-            case TRANSPORT:
-                return apiUsageState.getTransportState();
-            case RE:
-                return apiUsageState.getReExecState();
-            case DB:
-                return apiUsageState.getDbStorageState();
-            case JS:
-                return apiUsageState.getJsExecState();
-            case EMAIL:
-                return apiUsageState.getEmailExecState();
-            case SMS:
-                return apiUsageState.getSmsExecState();
-            default:
-                return ApiUsageStateValue.ENABLED;
+    private Pair<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(ApiFeature feature) {
+        ApiUsageStateValue featureValue = ApiUsageStateValue.ENABLED;
+        for (ApiUsageRecordKey recordKey : ApiUsageRecordKey.getKeys(feature)) {
+            long value = get(recordKey);
+            long threshold = getProfileThreshold(recordKey);
+            long warnThreshold = getProfileWarnThreshold(recordKey);
+            ApiUsageStateValue tmpValue;
+            if (threshold == 0 || value == 0 || value < warnThreshold) {
+                tmpValue = ApiUsageStateValue.ENABLED;
+            } else if (value < threshold) {
+                tmpValue = ApiUsageStateValue.WARNING;
+            } else {
+                tmpValue = ApiUsageStateValue.DISABLED;
+            }
+            featureValue = ApiUsageStateValue.toMoreRestricted(featureValue, tmpValue);
         }
+        return setFeatureValue(feature, featureValue) ? Pair.of(feature, featureValue) : null;
     }
 
-    public boolean setFeatureValue(ApiFeature feature, ApiUsageStateValue value) {
-        ApiUsageStateValue currentValue = getFeatureValue(feature);
-        switch (feature) {
-            case TRANSPORT:
-                apiUsageState.setTransportState(value);
-                break;
-            case RE:
-                apiUsageState.setReExecState(value);
-                break;
-            case DB:
-                apiUsageState.setDbStorageState(value);
-                break;
-            case JS:
-                apiUsageState.setJsExecState(value);
-                break;
-            case EMAIL:
-                apiUsageState.setEmailExecState(value);
-                break;
-            case SMS:
-                apiUsageState.setSmsExecState(value);
-                break;
-        }
-        return !currentValue.equals(value);
-    }
 
     public Map<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThresholds() {
         return checkStateUpdatedDueToThreshold(new HashSet<>(Arrays.asList(ApiFeature.values())));
@@ -174,23 +94,9 @@ public class TenantApiUsageState {
         return result;
     }
 
-    public Pair<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(ApiFeature feature) {
-        ApiUsageStateValue featureValue = ApiUsageStateValue.ENABLED;
-        for (ApiUsageRecordKey recordKey : ApiUsageRecordKey.getKeys(feature)) {
-            long value = get(recordKey);
-            long threshold = getProfileThreshold(recordKey);
-            long warnThreshold = getProfileWarnThreshold(recordKey);
-            ApiUsageStateValue tmpValue;
-            if (threshold == 0 || value == 0 || value < warnThreshold) {
-                tmpValue = ApiUsageStateValue.ENABLED;
-            } else if (value < threshold) {
-                tmpValue = ApiUsageStateValue.WARNING;
-            } else {
-                tmpValue = ApiUsageStateValue.DISABLED;
-            }
-            featureValue = ApiUsageStateValue.toMoreRestricted(featureValue, tmpValue);
-        }
-        return setFeatureValue(feature, featureValue) ? Pair.of(feature, featureValue) : null;
+    @Override
+    public EntityType getEntityType() {
+        return EntityType.TENANT;
     }
 
 }

application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java

@@ -65,7 +65,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.locks.ReentrantLock;
 
 
 @Service
@@ -78,8 +77,6 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService {
     private static final String DEVICE_PROVISION_STATE = "provisionState";
     private static final String PROVISIONED_STATE = "provisioned";
 
-    private final ReentrantLock deviceCreationLock = new ReentrantLock();
-
     @Autowired
     DeviceDao deviceDao;
 
@@ -177,12 +174,7 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService {
     }
 
     private ProvisionResponse createDevice(ProvisionRequest provisionRequest, DeviceProfile profile) {
-        deviceCreationLock.lock();
-        try {
-            return processCreateDevice(provisionRequest, profile);
-        } finally {
-            deviceCreationLock.unlock();
-        }
+        return processCreateDevice(provisionRequest, profile);
     }
 
     private void notify(Device device, ProvisionRequest provisionRequest, String type, boolean success) {
@@ -191,28 +183,26 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService {
     }
 
     private ProvisionResponse processCreateDevice(ProvisionRequest provisionRequest, DeviceProfile profile) {
-        Device device = deviceService.findDeviceByTenantIdAndName(profile.getTenantId(), provisionRequest.getDeviceName());
         try {
-            if (device == null) {
-                if (StringUtils.isEmpty(provisionRequest.getDeviceName())) {
-                    String newDeviceName = RandomStringUtils.randomAlphanumeric(20);
-                    log.info("Device name not found in provision request. Generated name is: {}", newDeviceName);
-                    provisionRequest.setDeviceName(newDeviceName);
-                }
-                Device savedDevice = deviceService.saveDevice(provisionRequest, profile);
-
-                deviceStateService.onDeviceAdded(savedDevice);
-                saveProvisionStateAttribute(savedDevice).get();
-                pushDeviceCreatedEventToRuleEngine(savedDevice);
-                notify(savedDevice, provisionRequest, DataConstants.PROVISION_SUCCESS, true);
-
-                return new ProvisionResponse(getDeviceCredentials(savedDevice), ProvisionResponseStatus.SUCCESS);
-            } else {
-                log.warn("[{}] The device is already provisioned!", device.getName());
+            if (StringUtils.isEmpty(provisionRequest.getDeviceName())) {
+                String newDeviceName = RandomStringUtils.randomAlphanumeric(20);
+                log.info("Device name not found in provision request. Generated name is: {}", newDeviceName);
+                provisionRequest.setDeviceName(newDeviceName);
+            }
+            Device savedDevice = deviceService.saveDevice(provisionRequest, profile);
+
+            deviceStateService.onDeviceAdded(savedDevice);
+            saveProvisionStateAttribute(savedDevice).get();
+            pushDeviceCreatedEventToRuleEngine(savedDevice);
+            notify(savedDevice, provisionRequest, DataConstants.PROVISION_SUCCESS, true);
+
+            return new ProvisionResponse(getDeviceCredentials(savedDevice), ProvisionResponseStatus.SUCCESS);
+        } catch (Exception e) {
+            log.warn("[{}] Error during device creation from provision request: [{}]", provisionRequest.getDeviceName(), provisionRequest, e);
+            Device device = deviceService.findDeviceByTenantIdAndName(profile.getTenantId(), provisionRequest.getDeviceName());
+            if (device != null) {
                 notify(device, provisionRequest, DataConstants.PROVISION_FAILURE, false);
-                throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name());
             }
-        } catch (InterruptedException | ExecutionException e) {
             throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name());
         }
     }
@@ -230,7 +220,7 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService {
     private void pushProvisionEventToRuleEngine(ProvisionRequest request, Device device, String type) {
         try {
             JsonNode entityNode = JacksonUtil.valueToTree(request);
-            TbMsg msg = TbMsg.newMsg(type, device.getId(), createTbMsgMetaData(device), JacksonUtil.toString(entityNode));
+            TbMsg msg = TbMsg.newMsg(type, device.getId(), device.getCustomerId(), createTbMsgMetaData(device), JacksonUtil.toString(entityNode));
             sendToRuleEngine(device.getTenantId(), msg, null);
         } catch (IllegalArgumentException e) {
             log.warn("[{}] Failed to push device action to rule engine: {}", device.getId(), type, e);
@@ -240,7 +230,7 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService {
     private void pushDeviceCreatedEventToRuleEngine(Device device) {
         try {
             ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.valueToTree(device);
-            TbMsg msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, device.getId(), createTbMsgMetaData(device), JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode));
+            TbMsg msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, device.getId(), device.getCustomerId(), createTbMsgMetaData(device), JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode));
             sendToRuleEngine(device.getTenantId(), msg, null);
         } catch (JsonProcessingException | IllegalArgumentException e) {
             log.warn("[{}] Failed to push device action to rule engine: {}", device.getId(), DataConstants.ENTITY_CREATED, e);

application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java

@@ -172,7 +172,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
     @Override
     public void onEdgeEvent(EdgeId edgeId) {
         log.trace("[{}] onEdgeEvent", edgeId.getId());
-        if (!sessionNewEvents.get(edgeId)) {
+        if (Boolean.FALSE.equals(sessionNewEvents.get(edgeId))) {
             log.trace("[{}] set session new events flag to true", edgeId.getId());
             sessionNewEvents.put(edgeId, true);
         }
@@ -204,7 +204,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
         if (sessions.containsKey(edgeId)) {
             ScheduledFuture<?> schedule = scheduler.schedule(() -> {
                 try {
-                    if (sessionNewEvents.get(edgeId)) {
+                    if (Boolean.TRUE.equals(sessionNewEvents.get(edgeId))) {
                         log.trace("[{}] Set session new events flag to false", edgeId.getId());
                         sessionNewEvents.put(edgeId, false);
                         session.processEdgeEvents();

application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java

@@ -920,7 +920,7 @@ public final class EdgeGrpcSession implements Closeable {
         try {
             if (uplinkMsg.getEntityDataCount() > 0) {
                 for (EntityDataProto entityData : uplinkMsg.getEntityDataList()) {
-                    result.addAll(ctx.getTelemetryProcessor().onTelemetryUpdate(edge.getTenantId(), entityData));
+                    result.addAll(ctx.getTelemetryProcessor().onTelemetryUpdate(edge.getTenantId(), edge.getCustomerId(), entityData));
                 }
             }
             if (uplinkMsg.getDeviceUpdateMsgCount() > 0) {

application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java

@@ -40,8 +40,6 @@ public class DeviceProfileMsgConstructor {
                 .setName(deviceProfile.getName())
                 .setDefault(deviceProfile.isDefault())
                 .setType(deviceProfile.getType().name())
-                .setTransportType(deviceProfile.getTransportType().name())
-                .setProvisionType(deviceProfile.getProvisionType().name())
                 .setProfileDataBytes(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile.getProfileData())));
         // TODO: voba - should this be always null at the moment??
 //        if (deviceProfile.getDefaultRuleChainId() != null) {
@@ -54,6 +52,12 @@ public class DeviceProfileMsgConstructor {
         if (deviceProfile.getDescription() != null) {
             builder.setDescription(deviceProfile.getDescription());
         }
+        if (deviceProfile.getTransportType() != null) {
+            builder.setTransportType(deviceProfile.getTransportType().name());
+        }
+        if (deviceProfile.getProvisionType() != null) {
+            builder.setProvisionType(deviceProfile.getProvisionType().name());
+        }
         if (deviceProfile.getProvisionDeviceKey() != null) {
             builder.setProvisionDeviceKey(deviceProfile.getProvisionDeviceKey());
         }

application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceProcessor.java

@@ -231,7 +231,7 @@ public class DeviceProcessor extends BaseProcessor {
         try {
             DeviceId deviceId = device.getId();
             ObjectNode entityNode = mapper.valueToTree(device);
-            TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId,
+            TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId, device.getCustomerId(),
                     getActionTbMsgMetaData(edge, device.getCustomerId()), TbMsgDataType.JSON, mapper.writeValueAsString(entityNode));
             tbClusterService.pushMsgToRuleEngine(tenantId, deviceId, tbMsg, new TbQueueCallback() {
                 @Override

application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryProcessor.java

@@ -70,7 +70,7 @@ public class TelemetryProcessor extends BaseProcessor {
 
     private final Gson gson = new Gson();
 
-    public List<ListenableFuture<Void>> onTelemetryUpdate(TenantId tenantId, EntityDataProto entityData) {
+    public List<ListenableFuture<Void>> onTelemetryUpdate(TenantId tenantId, CustomerId customerId, EntityDataProto entityData) {
         log.trace("[{}] onTelemetryUpdate [{}]", tenantId, entityData);
         List<ListenableFuture<Void>> result = new ArrayList<>();
         EntityId entityId = constructEntityId(entityData);
@@ -80,14 +80,14 @@ public class TelemetryProcessor extends BaseProcessor {
             TbMsgMetaData metaData = new TbMsgMetaData();
             metaData.putValue(DataConstants.MSG_SOURCE_KEY, DataConstants.EDGE_MSG_SOURCE);
             if (entityData.hasPostAttributesMsg()) {
-                result.add(processPostAttributes(tenantId, entityId, entityData.getPostAttributesMsg(), metaData));
+                result.add(processPostAttributes(tenantId, customerId, entityId, entityData.getPostAttributesMsg(), metaData));
             }
             if (entityData.hasAttributesUpdatedMsg()) {
                 metaData.putValue("scope", entityData.getPostAttributeScope());
-                result.add(processAttributesUpdate(tenantId, entityId, entityData.getAttributesUpdatedMsg(), metaData));
+                result.add(processAttributesUpdate(tenantId, customerId, entityId, entityData.getAttributesUpdatedMsg(), metaData));
             }
             if (entityData.hasPostTelemetryMsg()) {
-                result.add(processPostTelemetry(tenantId, entityId, entityData.getPostTelemetryMsg(), metaData));
+                result.add(processPostTelemetry(tenantId, customerId, entityId, entityData.getPostTelemetryMsg(), metaData));
             }
         }
         if (entityData.hasAttributeDeleteMsg()) {
@@ -148,7 +148,7 @@ public class TelemetryProcessor extends BaseProcessor {
         }
     }
 
-    private ListenableFuture<Void> processPostTelemetry(TenantId tenantId, EntityId entityId, TransportProtos.PostTelemetryMsg msg, TbMsgMetaData metaData) {
+    private ListenableFuture<Void> processPostTelemetry(TenantId tenantId, CustomerId customerId, EntityId entityId, TransportProtos.PostTelemetryMsg msg, TbMsgMetaData metaData) {
         SettableFuture<Void> futureToSet = SettableFuture.create();
         for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) {
             JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList());
@@ -156,7 +156,7 @@ public class TelemetryProcessor extends BaseProcessor {
             Pair<String, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
             String queueName = defaultQueueAndRuleChain.getKey();
             RuleChainId ruleChainId = defaultQueueAndRuleChain.getValue();
-            TbMsg tbMsg = TbMsg.newMsg(queueName, SessionMsgType.POST_TELEMETRY_REQUEST.name(), entityId, metaData, gson.toJson(json), ruleChainId, null);
+            TbMsg tbMsg = TbMsg.newMsg(queueName, SessionMsgType.POST_TELEMETRY_REQUEST.name(), entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
             tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() {
                 @Override
                 public void onSuccess(TbQueueMsgMetadata metadata) {
@@ -173,13 +173,13 @@ public class TelemetryProcessor extends BaseProcessor {
         return futureToSet;
     }
 
-    private ListenableFuture<Void> processPostAttributes(TenantId tenantId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData) {
+    private ListenableFuture<Void> processPostAttributes(TenantId tenantId, CustomerId customerId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData) {
         SettableFuture<Void> futureToSet = SettableFuture.create();
         JsonObject json = JsonUtils.getJsonObject(msg.getKvList());
         Pair<String, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
         String queueName = defaultQueueAndRuleChain.getKey();
         RuleChainId ruleChainId = defaultQueueAndRuleChain.getValue();
-        TbMsg tbMsg = TbMsg.newMsg(queueName, SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), entityId, metaData, gson.toJson(json), ruleChainId, null);
+        TbMsg tbMsg = TbMsg.newMsg(queueName, SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
         tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() {
             @Override
             public void onSuccess(TbQueueMsgMetadata metadata) {
@@ -195,7 +195,7 @@ public class TelemetryProcessor extends BaseProcessor {
         return futureToSet;
     }
 
-    private ListenableFuture<Void> processAttributesUpdate(TenantId tenantId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData) {
+    private ListenableFuture<Void> processAttributesUpdate(TenantId tenantId, CustomerId customerId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData) {
         SettableFuture<Void> futureToSet = SettableFuture.create();
         JsonObject json = JsonUtils.getJsonObject(msg.getKvList());
         Set<AttributeKvEntry> attributes = JsonConverter.convertToAttributes(json);
@@ -206,7 +206,7 @@ public class TelemetryProcessor extends BaseProcessor {
                 Pair<String, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
                 String queueName = defaultQueueAndRuleChain.getKey();
                 RuleChainId ruleChainId = defaultQueueAndRuleChain.getValue();
-                TbMsg tbMsg = TbMsg.newMsg(queueName, DataConstants.ATTRIBUTES_UPDATED, entityId, metaData, gson.toJson(json), ruleChainId, null);
+                TbMsg tbMsg = TbMsg.newMsg(queueName, DataConstants.ATTRIBUTES_UPDATED, entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
                 tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() {
                     @Override
                     public void onSuccess(TbQueueMsgMetadata metadata) {

application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java

@@ -51,6 +51,7 @@ public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabase
             case "2.5.0":
             case "3.1.1":
             case "3.2.1":
+            case "3.2.2":
                 break;
             default:
                 throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion);

application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@@ -444,11 +444,6 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
         installScripts.loadSystemWidgets();
     }
 
-    @Override
-    public void loadSystemLwm2mResources() throws Exception {
-        installScripts.loadSystemLwm2mResources();
-    }
-
     private User createUser(Authority authority,
                             TenantId tenantId,
                             CustomerId customerId,

application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@@ -22,8 +22,6 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.springframework.util.StringUtils;
 import org.thingsboard.server.common.data.Dashboard;
-import org.thingsboard.server.common.data.ResourceType;
-import org.thingsboard.server.common.data.TbResource;
 import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
@@ -33,9 +31,8 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
 import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
 import org.thingsboard.server.common.data.widget.WidgetsBundle;
 import org.thingsboard.server.dao.dashboard.DashboardService;
-import org.thingsboard.server.dao.exception.DataValidationException;
 import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
-import org.thingsboard.server.dao.resource.TbResourceService;
+import org.thingsboard.server.dao.resource.ResourceService;
 import org.thingsboard.server.dao.rule.RuleChainService;
 import org.thingsboard.server.dao.widget.WidgetTypeService;
 import org.thingsboard.server.dao.widget.WidgetsBundleService;
@@ -45,7 +42,6 @@ import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Base64;
 import java.util.Optional;
 
 import static org.thingsboard.server.service.install.DatabaseHelper.objectMapper;
@@ -97,7 +93,7 @@ public class InstallScripts {
     private OAuth2ConfigTemplateService oAuth2TemplateService;
 
     @Autowired
-    private TbResourceService resourceService;
+    private ResourceService resourceService;
 
     private Path getTenantRuleChainsDir() {
         return Paths.get(getDataDir(), JSON_DIR, TENANT_DIR, RULE_CHAINS_DIR);
@@ -210,29 +206,6 @@ public class InstallScripts {
         }
     }
 
-    public void loadSystemLwm2mResources() throws Exception {
-        Path modelsDir = Paths.get(getDataDir(), MODELS_DIR);
-        if (Files.isDirectory(modelsDir)) {
-            try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(modelsDir, path -> path.toString().endsWith(XML_EXT))) {
-                dirStream.forEach(
-                        path -> {
-                            try {
-                                byte[] fileBytes = Files.readAllBytes(path);
-                                TbResource resource = new TbResource();
-                                resource.setFileName(path.getFileName().toString());
-                                resource.setTenantId(TenantId.SYS_TENANT_ID);
-                                resource.setResourceType(ResourceType.LWM2M_MODEL);
-                                resource.setData(Base64.getEncoder().encodeToString(fileBytes));
-                                resourceService.saveResource(resource);
-                            } catch (Exception e) {
-                                throw new DataValidationException(String.format("Could not parse the XML of objectModel with name %s", path.toString()));
-                            }
-                        }
-                );
-            }
-        }
-    }
-
     public void loadDashboards(TenantId tenantId, CustomerId customerId) throws Exception {
         Path dashboardsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, DASHBOARDS_DIR);
         try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(dashboardsDir, path -> path.toString().endsWith(JSON_EXT))) {
@@ -259,18 +232,12 @@ public class InstallScripts {
         try {
             createDefaultRuleChains(tenantId);
             createDefaultRuleChain(tenantId, "Thermostat");
-            loadEdgeDemoRuleChains(tenantId);
         } catch (Exception e) {
             log.error("Unable to load dashboard from json", e);
             throw new RuntimeException("Unable to load dashboard from json", e);
         }
     }
 
-    private void loadEdgeDemoRuleChains(TenantId tenantId) throws Exception {
-        Path edgeDemoRuleChainsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, EDGE_MANAGEMENT, RULE_CHAINS_DIR);
-        loadRuleChainsFromPath(tenantId, edgeDemoRuleChainsDir);
-    }
-
     public void createOAuth2Templates() throws Exception {
         Path oauth2ConfigTemplatesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, OAUTH2_CONFIG_TEMPLATES_DIR);
         try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(oauth2ConfigTemplatesDir, path -> path.toString().endsWith(JSON_EXT))) {

application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java

@@ -209,6 +209,12 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
                     executeQuery(conn, "DROP FUNCTION IF EXISTS delete_customer_records_from_ts_kv(character varying, character varying, bigint);");
                 }
                 break;
+            case "3.2.2":
+                try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
+                    log.info("Load Drop Partitions functions ...");
+                    loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL, "2.4.3");
+                }
+                break;
             default:
                 throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
         }

application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@@ -390,7 +390,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
                             pageData = tenantService.findTenants(pageLink);
                             for (Tenant tenant : pageData.getData()) {
                                 try {
-                                    apiUsageStateService.createDefaultApiUsageState(tenant.getId());
+                                    apiUsageStateService.createDefaultApiUsageState(tenant.getId(), null);
                                 } catch (Exception e) {
                                 }
                                 List<EntitySubtype> deviceTypes = deviceService.findDeviceTypesByTenantId(tenant.getId()).get();
@@ -450,37 +450,19 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
                 try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
                     log.info("Updating schema ...");
                     try {
-                        conn.createStatement().execute("CREATE TABLE IF NOT EXISTS resource ( " +
-                                "id uuid NOT NULL CONSTRAINT resource_pkey PRIMARY KEY, " +
-                                "created_time bigint NOT NULL, " +
-                                "tenant_id uuid NOT NULL, " +
-                                "title varchar(255) NOT NULL, " +
-                                "resource_type varchar(32) NOT NULL, " +
-                                "resource_key varchar(255) NOT NULL, " +
-                                "search_text varchar(255), " +
-                                "file_name varchar(255) NOT NULL, " +
-                                "data varchar, " +
-                                "CONSTRAINT resource_unq_key UNIQUE (tenant_id, resource_type, resource_key)" +
-                                ");");
-
-                        conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3003000;");
-                        installScripts.loadSystemLwm2mResources();
-
-                        schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", SCHEMA_UPDATE_SQL);
-                        loadSql(schemaUpdateFile, conn);
-                        try {
-                            conn.createStatement().execute("ALTER TABLE rule_chain ADD COLUMN type varchar(255) DEFAULT 'CORE'"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
-                        } catch (Exception ignored) {}
-
-                        log.info("Load Edge TTL functions ...");
-                        schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", "schema_update_ttl.sql");
-                        loadSql(schemaUpdateFile, conn);
-                        log.info("Edge TTL functions successfully loaded!");
-
-                    } catch (Exception e) {
-                        log.error("Failed updating schema!!!", e);
+                        conn.createStatement().execute("ALTER TABLE rule_chain ADD COLUMN type varchar(255) DEFAULT 'CORE'"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
+                    } catch (Exception ignored) {
                     }
+                    schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", SCHEMA_UPDATE_SQL);
+                    loadSql(schemaUpdateFile, conn);
+                    log.info("Load Edge TTL functions ...");
+                    schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", "schema_update_ttl.sql");
+                    loadSql(schemaUpdateFile, conn);
+                    log.info("Edge TTL functions successfully loaded!");
+                    conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3003000;");
                     log.info("Schema updated.");
+                } catch (Exception e) {
+                    log.error("Failed updating schema!!!", e);
                 }
                 break;
             default:

application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java

@@ -33,6 +33,4 @@ public interface SystemDataLoaderService {
 
     void deleteSystemWidgetBundle(String bundleAlias) throws Exception;
 
-    void loadSystemLwm2mResources() throws Exception;
-
 }

application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java

@@ -184,6 +184,8 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
                     loadSql(conn, LOAD_TTL_FUNCTIONS_SQL, "3.2.1");
                 }
                 break;
+            case "3.2.2":
+                break;
             default:
                 throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
         }

application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@@ -23,25 +23,36 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.rule.engine.profile.TbDeviceProfileNode;
 import org.thingsboard.rule.engine.profile.TbDeviceProfileNodeConfiguration;
 import org.thingsboard.server.common.data.EntityView;
 import org.thingsboard.server.common.data.Tenant;
+import org.thingsboard.server.common.data.alarm.Alarm;
+import org.thingsboard.server.common.data.alarm.AlarmInfo;
+import org.thingsboard.server.common.data.alarm.AlarmQuery;
 import org.thingsboard.server.common.data.id.EntityViewId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
 import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
 import org.thingsboard.server.common.data.kv.TsKvEntry;
+import org.thingsboard.server.common.data.oauth2.OAuth2Info;
+import org.thingsboard.server.common.data.oauth2.deprecated.OAuth2ClientsParams;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.page.PageLink;
+import org.thingsboard.server.common.data.page.TimePageLink;
 import org.thingsboard.server.common.data.rule.RuleChain;
 import org.thingsboard.server.common.data.rule.RuleChainMetaData;
 import org.thingsboard.server.common.data.rule.RuleNode;
+import org.thingsboard.server.dao.alarm.AlarmDao;
+import org.thingsboard.server.dao.alarm.AlarmService;
+import org.thingsboard.server.dao.entity.EntityService;
 import org.thingsboard.server.dao.entityview.EntityViewService;
+import org.thingsboard.server.dao.oauth2.OAuth2Service;
+import org.thingsboard.server.dao.oauth2.OAuth2Utils;
 import org.thingsboard.server.dao.rule.RuleChainService;
 import org.thingsboard.server.dao.tenant.TenantService;
 import org.thingsboard.server.dao.timeseries.TimeseriesService;
-import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.service.install.InstallScripts;
 
 import java.util.ArrayList;
@@ -72,6 +83,18 @@ public class DefaultDataUpdateService implements DataUpdateService {
     @Autowired
     private TimeseriesService tsService;
 
+    @Autowired
+    private AlarmService alarmService;
+
+    @Autowired
+    private EntityService entityService;
+
+    @Autowired
+    private AlarmDao alarmDao;
+
+    @Autowired
+    private OAuth2Service oAuth2Service;
+
     @Override
     public void updateData(String fromVersion) throws Exception {
         switch (fromVersion) {
@@ -90,14 +113,26 @@ public class DefaultDataUpdateService implements DataUpdateService {
             case "3.2.2":
                 log.info("Updating data from version 3.2.2 to 3.3.0 ...");
                 tenantsDefaultEdgeRuleChainUpdater.updateEntities(null);
+                tenantsAlarmsCustomerUpdater.updateEntities(null);
+                updateOAuth2Params();
                 break;
             default:
                 throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion);
         }
     }
 
-    private PaginatedUpdater<String, Tenant> tenantsDefaultRuleChainUpdater =
-            new PaginatedUpdater<String, Tenant>() {
+    private final PaginatedUpdater<String, Tenant> tenantsDefaultRuleChainUpdater =
+            new PaginatedUpdater<>() {
+
+                @Override
+                protected String getName() {
+                    return "Tenants default rule chain updater";
+                }
+
+                @Override
+                protected boolean forceReportTotal() {
+                    return true;
+                }
 
                 @Override
                 protected PageData<Tenant> findEntities(String region, PageLink pageLink) {
@@ -117,8 +152,18 @@ public class DefaultDataUpdateService implements DataUpdateService {
                 }
             };
 
-    private PaginatedUpdater<String, Tenant> tenantsDefaultEdgeRuleChainUpdater =
-            new PaginatedUpdater<String, Tenant>() {
+    private final PaginatedUpdater<String, Tenant> tenantsDefaultEdgeRuleChainUpdater =
+            new PaginatedUpdater<>() {
+
+                @Override
+                protected String getName() {
+                    return "Tenants default edge rule chain updater";
+                }
+
+                @Override
+                protected boolean forceReportTotal() {
+                    return true;
+                }
 
                 @Override
                 protected PageData<Tenant> findEntities(String region, PageLink pageLink) {
@@ -138,8 +183,18 @@ public class DefaultDataUpdateService implements DataUpdateService {
                 }
             };
 
-    private PaginatedUpdater<String, Tenant> tenantsRootRuleChainUpdater =
-            new PaginatedUpdater<String, Tenant>() {
+    private final PaginatedUpdater<String, Tenant> tenantsRootRuleChainUpdater =
+            new PaginatedUpdater<>() {
+
+                @Override
+                protected String getName() {
+                    return "Tenants root rule chain updater";
+                }
+
+                @Override
+                protected boolean forceReportTotal() {
+                    return true;
+                }
 
                 @Override
                 protected PageData<Tenant> findEntities(String region, PageLink pageLink) {
@@ -192,8 +247,18 @@ public class DefaultDataUpdateService implements DataUpdateService {
                 }
             };
 
-    private PaginatedUpdater<String, Tenant> tenantsEntityViewsUpdater =
-            new PaginatedUpdater<String, Tenant>() {
+    private final PaginatedUpdater<String, Tenant> tenantsEntityViewsUpdater =
+            new PaginatedUpdater<>() {
+
+                @Override
+                protected String getName() {
+                    return "Tenants entity views updater";
+                }
+
+                @Override
+                protected boolean forceReportTotal() {
+                    return true;
+                }
 
                 @Override
                 protected PageData<Tenant> findEntities(String region, PageLink pageLink) {
@@ -261,4 +326,64 @@ public class DefaultDataUpdateService implements DataUpdateService {
         }, MoreExecutors.directExecutor());
     }
 
+    private final PaginatedUpdater<String, Tenant> tenantsAlarmsCustomerUpdater =
+            new PaginatedUpdater<>() {
+
+                @Override
+                protected String getName() {
+                    return "Tenants alarms customer updater";
+                }
+
+                @Override
+                protected boolean forceReportTotal() {
+                    return true;
+                }
+
+                @Override
+                protected PageData<Tenant> findEntities(String region, PageLink pageLink) {
+                    return tenantService.findTenants(pageLink);
+                }
+
+                @Override
+                protected void updateEntity(Tenant tenant) {
+                    updateTenantAlarmsCustomer(tenant.getId());
+                }
+            };
+
+    private void updateTenantAlarmsCustomer(TenantId tenantId) {
+        AlarmQuery alarmQuery = new AlarmQuery(null, new TimePageLink(100), null, null, false);
+        PageData<AlarmInfo> alarms = alarmDao.findAlarms(tenantId, alarmQuery);
+        boolean hasNext = true;
+        while (hasNext) {
+            for (Alarm alarm : alarms.getData()) {
+                if (alarm.getCustomerId() == null && alarm.getOriginator() != null) {
+                    alarm.setCustomerId(entityService.fetchEntityCustomerId(tenantId, alarm.getOriginator()));
+                    alarmDao.save(tenantId, alarm);
+                }
+            }
+            if (alarms.hasNext()) {
+                alarmQuery.setPageLink(alarmQuery.getPageLink().nextPageLink());
+                alarms = alarmDao.findAlarms(tenantId, alarmQuery);
+            } else {
+                hasNext = false;
+            }
+        }
+    }
+
+    private void updateOAuth2Params() {
+        try {
+            OAuth2ClientsParams oauth2ClientsParams = oAuth2Service.findOAuth2Params();
+            if (!oauth2ClientsParams.getDomainsParams().isEmpty()) {
+                log.info("Updating OAuth2 parameters ...");
+                OAuth2Info oAuth2Info = OAuth2Utils.clientParamsToOAuth2Info(oauth2ClientsParams);
+                oAuth2Service.saveOAuth2Info(oAuth2Info);
+                oAuth2Service.saveOAuth2Params(new OAuth2ClientsParams(false, Collections.emptyList()));
+                log.info("Successfully updated OAuth2 parameters!");
+            }
+        }
+        catch (Exception e) {
+           log.error("Failed to update OAuth2 parameters", e);
+        }
+    }
+
 }

application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java

@@ -15,16 +15,20 @@
  */
 package org.thingsboard.server.service.install.update;
 
+import lombok.extern.slf4j.Slf4j;
 import org.thingsboard.server.common.data.SearchTextBased;
 import org.thingsboard.server.common.data.id.UUIDBased;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.page.PageLink;
 
+@Slf4j
 public abstract class PaginatedUpdater<I, D extends SearchTextBased<? extends UUIDBased>> {
 
     private static final int DEFAULT_LIMIT = 100;
+    private int updated = 0;
 
     public void updateEntities(I id) {
+        updated = 0;
         PageLink pageLink = new PageLink(DEFAULT_LIMIT);
         boolean hasNext = true;
         while (hasNext) {
@@ -32,13 +36,25 @@ public abstract class PaginatedUpdater<I, D extends SearchTextBased<? extends UU
             for (D entity : entities.getData()) {
                 updateEntity(entity);
             }
+            updated += entities.getData().size();
             hasNext = entities.hasNext();
             if (hasNext) {
+                log.info("{}: {} entities updated so far...", getName(), updated);
                 pageLink = pageLink.nextPageLink();
+            } else {
+                if (updated > DEFAULT_LIMIT || forceReportTotal()) {
+                    log.info("{}: {} total entities updated.", getName(), updated);
+                }
             }
         }
     }
 
+    protected boolean forceReportTotal() {
+        return false;
+    }
+
+    protected abstract String getName();
+
     protected abstract PageData<D> findEntities(I id, PageLink pageLink);
 
     protected abstract void updateEntity(D entity);

application/src/main/java/org/thingsboard/server/service/lwm2m/LwM2MModelsRepository.java

@@ -1,173 +0,0 @@
-/**
- * Copyright © 2016-2021 The Thingsboard Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.thingsboard.server.service.lwm2m;
-
-
-import lombok.extern.slf4j.Slf4j;
-import org.eclipse.leshan.core.util.Hex;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
-import org.springframework.stereotype.Service;
-import org.thingsboard.server.common.data.lwm2m.ServerSecurityConfig;
-import org.thingsboard.server.common.transport.lwm2m.LwM2MTransportConfigBootstrap;
-import org.thingsboard.server.common.transport.lwm2m.LwM2MTransportConfigServer;
-import org.thingsboard.server.transport.lwm2m.secure.LwM2MSecurityMode;
-
-import java.math.BigInteger;
-import java.security.AlgorithmParameters;
-import java.security.GeneralSecurityException;
-import java.security.KeyFactory;
-import java.security.KeyStoreException;
-import java.security.PublicKey;
-import java.security.cert.CertificateEncodingException;
-import java.security.cert.X509Certificate;
-import java.security.spec.ECGenParameterSpec;
-import java.security.spec.ECParameterSpec;
-import java.security.spec.ECPoint;
-import java.security.spec.ECPublicKeySpec;
-import java.security.spec.KeySpec;
-
-@Slf4j
-@Service
-@ConditionalOnExpression("('${service.type:null}'=='tb-transport' && '${transport.lwm2m.enabled:false}'=='true') || '${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'")
-public class LwM2MModelsRepository {
-
-    private static final String INCORRECT_TENANT_ID = "Incorrect tenantId ";
-
-    @Autowired
-    LwM2MTransportConfigServer contextServer;
-
-
-    @Autowired
-    LwM2MTransportConfigBootstrap contextBootStrap;
-
-    /**
-     * @param securityMode
-     * @param bootstrapServerIs
-     * @return ServerSecurityConfig more value is default: Important - port, host, publicKey
-     */
-    public ServerSecurityConfig getBootstrapSecurityInfo(String securityMode, boolean bootstrapServerIs) {
-        LwM2MSecurityMode lwM2MSecurityMode = LwM2MSecurityMode.fromSecurityMode(securityMode.toLowerCase());
-        return getBootstrapServer(bootstrapServerIs, lwM2MSecurityMode);
-    }
-
-    /**
-     * @param bootstrapServerIs
-     * @param mode
-     * @return ServerSecurityConfig more value is default: Important - port, host, publicKey
-     */
-    private ServerSecurityConfig getBootstrapServer(boolean bootstrapServerIs, LwM2MSecurityMode mode) {
-        ServerSecurityConfig bsServ = new ServerSecurityConfig();
-        bsServ.setBootstrapServerIs(bootstrapServerIs);
-        if (bootstrapServerIs) {
-            bsServ.setServerId(contextBootStrap.getBootstrapServerId());
-            switch (mode) {
-                case NO_SEC:
-                    bsServ.setHost(contextBootStrap.getBootstrapHost());
-                    bsServ.setPort(contextBootStrap.getBootstrapPortNoSec());
-                    bsServ.setServerPublicKey("");
-                    break;
-                case PSK:
-                    bsServ.setHost(contextBootStrap.getBootstrapHostSecurity());
-                    bsServ.setPort(contextBootStrap.getBootstrapPortSecurity());
-                    bsServ.setServerPublicKey("");
-                    break;
-                case RPK:
-                case X509:
-                    bsServ.setHost(contextBootStrap.getBootstrapHostSecurity());
-                    bsServ.setPort(contextBootStrap.getBootstrapPortSecurity());
-                    bsServ.setServerPublicKey(getPublicKey (contextBootStrap.getBootstrapAlias(), this.contextBootStrap.getBootstrapPublicX(), this.contextBootStrap.getBootstrapPublicY()));
-                    break;
-                default:
-                    break;
-            }
-        } else {
-            bsServ.setServerId(contextServer.getServerId());
-            switch (mode) {
-                case NO_SEC:
-                    bsServ.setHost(contextServer.getServerHost());
-                    bsServ.setPort(contextServer.getServerPortNoSec());
-                    bsServ.setServerPublicKey("");
-                    break;
-                case PSK:
-                    bsServ.setHost(contextServer.getServerHostSecurity());
-                    bsServ.setPort(contextServer.getServerPortSecurity());
-                    bsServ.setServerPublicKey("");
-                    break;
-                case RPK:
-                case X509:
-                    bsServ.setHost(contextServer.getServerHostSecurity());
-                    bsServ.setPort(contextServer.getServerPortSecurity());
-                    bsServ.setServerPublicKey(getPublicKey (contextServer.getServerAlias(), this.contextServer.getServerPublicX(), this.contextServer.getServerPublicY()));
-                    break;
-                default:
-                    break;
-            }
-        }
-        return bsServ;
-    }
-
-    private String getPublicKey (String alias, String publicServerX, String publicServerY) {
-        String publicKey = getServerPublicKeyX509(alias);
-        return publicKey != null ? publicKey : getRPKPublicKey(publicServerX, publicServerY);
-    }
-
-    /**
-     * @param alias
-     * @return PublicKey format HexString or null
-     */
-    private String getServerPublicKeyX509(String alias) {
-        try {
-            X509Certificate serverCertificate = (X509Certificate) contextServer.getKeyStoreValue().getCertificate(alias);
-            return Hex.encodeHexString(serverCertificate.getEncoded());
-        } catch (CertificateEncodingException | KeyStoreException e) {
-            e.printStackTrace();
-        }
-        return null;
-    }
-
-    /**
-     * @param publicServerX
-     * @param publicServerY
-     * @return PublicKey format HexString or null
-     */
-    private String getRPKPublicKey(String publicServerX, String publicServerY) {
-        try {
-            /** Get Elliptic Curve Parameter spec for secp256r1 */
-            AlgorithmParameters algoParameters = AlgorithmParameters.getInstance("EC");
-            algoParameters.init(new ECGenParameterSpec("secp256r1"));
-            ECParameterSpec parameterSpec = algoParameters.getParameterSpec(ECParameterSpec.class);
-            if (publicServerX != null && !publicServerX.isEmpty() && publicServerY != null && !publicServerY.isEmpty()) {
-                /** Get point values */
-                byte[] publicX = Hex.decodeHex(publicServerX.toCharArray());
-                byte[] publicY = Hex.decodeHex(publicServerY.toCharArray());
-                /** Create key specs */
-                KeySpec publicKeySpec = new ECPublicKeySpec(new ECPoint(new BigInteger(publicX), new BigInteger(publicY)),
-                        parameterSpec);
-                /** Get keys */
-                PublicKey publicKey = KeyFactory.getInstance("EC").generatePublic(publicKeySpec);
-                if (publicKey != null && publicKey.getEncoded().length > 0) {
-                    return Hex.encodeHexString(publicKey.getEncoded());
-                }
-            }
-        } catch (GeneralSecurityException | IllegalArgumentException e) {
-            log.error("[{}] Failed generate Server RPK for profile", e.getMessage());
-            throw new RuntimeException(e);
-        }
-        return null;
-    }
-}
-

application/src/main/java/org/thingsboard/server/service/lwm2m/LwM2MServerSecurityInfoRepository.java

@@ -0,0 +1,110 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.lwm2m;
+
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.eclipse.leshan.core.util.Hex;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.common.data.lwm2m.ServerSecurityConfig;
+import org.thingsboard.server.transport.lwm2m.config.LwM2MSecureServerConfig;
+import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportBootstrapConfig;
+import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig;
+
+import java.math.BigInteger;
+import java.security.AlgorithmParameters;
+import java.security.GeneralSecurityException;
+import java.security.KeyFactory;
+import java.security.KeyStoreException;
+import java.security.PublicKey;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.security.spec.ECGenParameterSpec;
+import java.security.spec.ECParameterSpec;
+import java.security.spec.ECPoint;
+import java.security.spec.ECPublicKeySpec;
+import java.security.spec.KeySpec;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@ConditionalOnExpression("('${service.type:null}'=='tb-transport' && '${transport.lwm2m.enabled:false}'=='true') || '${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'")
+public class LwM2MServerSecurityInfoRepository {
+
+    private final LwM2MTransportServerConfig serverConfig;
+    private final LwM2MTransportBootstrapConfig bootstrapConfig;
+
+    public ServerSecurityConfig getServerSecurityInfo(boolean bootstrapServer) {
+        ServerSecurityConfig result = getServerSecurityConfig(bootstrapServer ? bootstrapConfig : serverConfig);
+        result.setBootstrapServerIs(bootstrapServer);
+        return result;
+    }
+
+    private ServerSecurityConfig getServerSecurityConfig(LwM2MSecureServerConfig serverConfig) {
+        ServerSecurityConfig bsServ = new ServerSecurityConfig();
+        bsServ.setServerId(serverConfig.getId());
+        bsServ.setHost(serverConfig.getHost());
+        bsServ.setPort(serverConfig.getPort());
+        bsServ.setSecurityHost(serverConfig.getSecureHost());
+        bsServ.setSecurityPort(serverConfig.getSecurePort());
+        bsServ.setServerPublicKey(getPublicKey(serverConfig.getCertificateAlias(), this.serverConfig.getPublicX(), this.serverConfig.getPublicY()));
+        return bsServ;
+    }
+
+    private String getPublicKey(String alias, String publicServerX, String publicServerY) {
+        String publicKey = getServerPublicKeyX509(alias);
+        return publicKey != null ? publicKey : getRPKPublicKey(publicServerX, publicServerY);
+    }
+
+    private String getServerPublicKeyX509(String alias) {
+        try {
+            X509Certificate serverCertificate = (X509Certificate) serverConfig.getKeyStoreValue().getCertificate(alias);
+            return Hex.encodeHexString(serverCertificate.getEncoded());
+        } catch (CertificateEncodingException | KeyStoreException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+    private String getRPKPublicKey(String publicServerX, String publicServerY) {
+        try {
+            /** Get Elliptic Curve Parameter spec for secp256r1 */
+            AlgorithmParameters algoParameters = AlgorithmParameters.getInstance("EC");
+            algoParameters.init(new ECGenParameterSpec("secp256r1"));
+            ECParameterSpec parameterSpec = algoParameters.getParameterSpec(ECParameterSpec.class);
+            if (publicServerX != null && !publicServerX.isEmpty() && publicServerY != null && !publicServerY.isEmpty()) {
+                /** Get point values */
+                byte[] publicX = Hex.decodeHex(publicServerX.toCharArray());
+                byte[] publicY = Hex.decodeHex(publicServerY.toCharArray());
+                /** Create key specs */
+                KeySpec publicKeySpec = new ECPublicKeySpec(new ECPoint(new BigInteger(publicX), new BigInteger(publicY)),
+                        parameterSpec);
+                /** Get keys */
+                PublicKey publicKey = KeyFactory.getInstance("EC").generatePublic(publicKeySpec);
+                if (publicKey != null && publicKey.getEncoded().length > 0) {
+                    return Hex.encodeHexString(publicKey.getEncoded());
+                }
+            }
+        } catch (GeneralSecurityException | IllegalArgumentException e) {
+            log.error("[{}] Failed generate Server RPK for profile", e.getMessage());
+            throw new RuntimeException(e);
+        }
+        return null;
+    }
+}
+

application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java

@@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.ApiUsageStateMailMessage;
 import org.thingsboard.server.common.data.ApiUsageStateValue;
 import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
+import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.dao.exception.IncorrectParameterException;
@@ -233,7 +234,7 @@ public class DefaultMailService implements MailService {
     }
 
     @Override
-    public void send(TenantId tenantId, String from, String to, String cc, String bcc, String subject, String body) throws MessagingException {
+    public void send(TenantId tenantId, CustomerId customerId, String from, String to, String cc, String bcc, String subject, String body) throws MessagingException {
         if (apiUsageStateService.getApiUsageState(tenantId).isEmailSendEnabled()) {
             MimeMessage mailMsg = mailSender.createMimeMessage();
             MimeMessageHelper helper = new MimeMessageHelper(mailMsg, "UTF-8");
@@ -248,7 +249,7 @@ public class DefaultMailService implements MailService {
             helper.setSubject(subject);
             helper.setText(body);
             mailSender.send(helper.getMimeMessage());
-            apiUsageClient.report(tenantId, ApiUsageRecordKey.EMAIL_EXEC_COUNT, 1);
+            apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.EMAIL_EXEC_COUNT, 1);
         } else {
             throw new RuntimeException("Email sending is disabled due to API limits!");
         }
@@ -308,6 +309,8 @@ public class DefaultMailService implements MailService {
             case EMAIL:
             case SMS:
                 return "send";
+            case ALARM:
+                return "create";
             default:
                 throw new RuntimeException("Not implemented!");
         }
@@ -326,6 +329,8 @@ public class DefaultMailService implements MailService {
             case EMAIL:
             case SMS:
                 return "sent";
+            case ALARM:
+                return "created";
             default:
                 throw new RuntimeException("Not implemented!");
         }

application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java

@@ -0,0 +1,352 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.ota;
+
+import com.google.common.util.concurrent.FutureCallback;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
+import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg;
+import org.thingsboard.server.common.data.DataConstants;
+import org.thingsboard.server.common.data.Device;
+import org.thingsboard.server.common.data.DeviceProfile;
+import org.thingsboard.server.common.data.OtaPackageInfo;
+import org.thingsboard.server.common.data.StringUtils;
+import org.thingsboard.server.common.data.id.DeviceId;
+import org.thingsboard.server.common.data.id.OtaPackageId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.kv.AttributeKey;
+import org.thingsboard.server.common.data.kv.AttributeKvEntry;
+import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
+import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
+import org.thingsboard.server.common.data.kv.LongDataEntry;
+import org.thingsboard.server.common.data.kv.StringDataEntry;
+import org.thingsboard.server.common.data.kv.TsKvEntry;
+import org.thingsboard.server.common.data.ota.OtaPackageType;
+import org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus;
+import org.thingsboard.server.common.data.ota.OtaPackageUtil;
+import org.thingsboard.server.common.data.page.PageData;
+import org.thingsboard.server.common.data.page.PageLink;
+import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
+import org.thingsboard.server.dao.device.DeviceProfileService;
+import org.thingsboard.server.dao.device.DeviceService;
+import org.thingsboard.server.dao.ota.OtaPackageService;
+import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
+import org.thingsboard.server.queue.TbQueueProducer;
+import org.thingsboard.server.queue.common.TbProtoQueueMsg;
+import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
+import org.thingsboard.server.queue.util.TbCoreComponent;
+import org.thingsboard.server.service.queue.TbClusterService;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Consumer;
+
+import static org.thingsboard.server.common.data.ota.OtaPackageKey.CHECKSUM;
+import static org.thingsboard.server.common.data.ota.OtaPackageKey.CHECKSUM_ALGORITHM;
+import static org.thingsboard.server.common.data.ota.OtaPackageKey.SIZE;
+import static org.thingsboard.server.common.data.ota.OtaPackageKey.STATE;
+import static org.thingsboard.server.common.data.ota.OtaPackageKey.TITLE;
+import static org.thingsboard.server.common.data.ota.OtaPackageKey.TS;
+import static org.thingsboard.server.common.data.ota.OtaPackageKey.URL;
+import static org.thingsboard.server.common.data.ota.OtaPackageKey.VERSION;
+import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE;
+import static org.thingsboard.server.common.data.ota.OtaPackageType.SOFTWARE;
+import static org.thingsboard.server.common.data.ota.OtaPackageUtil.getAttributeKey;
+import static org.thingsboard.server.common.data.ota.OtaPackageUtil.getTargetTelemetryKey;
+import static org.thingsboard.server.common.data.ota.OtaPackageUtil.getTelemetryKey;
+
+@Slf4j
+@Service
+@TbCoreComponent
+public class DefaultOtaPackageStateService implements OtaPackageStateService {
+
+    private final TbClusterService tbClusterService;
+    private final OtaPackageService otaPackageService;
+    private final DeviceService deviceService;
+    private final DeviceProfileService deviceProfileService;
+    private final RuleEngineTelemetryService telemetryService;
+    private final TbQueueProducer<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> otaPackageStateMsgProducer;
+
+    public DefaultOtaPackageStateService(TbClusterService tbClusterService, OtaPackageService otaPackageService,
+                                         DeviceService deviceService,
+                                         DeviceProfileService deviceProfileService,
+                                         RuleEngineTelemetryService telemetryService,
+                                         TbCoreQueueFactory coreQueueFactory) {
+        this.tbClusterService = tbClusterService;
+        this.otaPackageService = otaPackageService;
+        this.deviceService = deviceService;
+        this.deviceProfileService = deviceProfileService;
+        this.telemetryService = telemetryService;
+        this.otaPackageStateMsgProducer = coreQueueFactory.createToOtaPackageStateServiceMsgProducer();
+    }
+
+    @Override
+    public void update(Device device, Device oldDevice) {
+        updateFirmware(device, oldDevice);
+        updateSoftware(device, oldDevice);
+    }
+
+    private void updateFirmware(Device device, Device oldDevice) {
+        OtaPackageId newFirmwareId = device.getFirmwareId();
+        if (newFirmwareId == null) {
+            DeviceProfile newDeviceProfile = deviceProfileService.findDeviceProfileById(device.getTenantId(), device.getDeviceProfileId());
+            newFirmwareId = newDeviceProfile.getFirmwareId();
+        }
+        if (oldDevice != null) {
+            if (newFirmwareId != null) {
+                OtaPackageId oldFirmwareId = oldDevice.getFirmwareId();
+                if (oldFirmwareId == null) {
+                    DeviceProfile oldDeviceProfile = deviceProfileService.findDeviceProfileById(oldDevice.getTenantId(), oldDevice.getDeviceProfileId());
+                    oldFirmwareId = oldDeviceProfile.getFirmwareId();
+                }
+                if (!newFirmwareId.equals(oldFirmwareId)) {
+                    // Device was updated and new firmware is different from previous firmware.
+                    send(device.getTenantId(), device.getId(), newFirmwareId, System.currentTimeMillis(), FIRMWARE);
+                }
+            } else {
+                // Device was updated and new firmware is not set.
+                remove(device, FIRMWARE);
+            }
+        } else if (newFirmwareId != null) {
+            // Device was created and firmware is defined.
+            send(device.getTenantId(), device.getId(), newFirmwareId, System.currentTimeMillis(), FIRMWARE);
+        }
+    }
+
+    private void updateSoftware(Device device, Device oldDevice) {
+        OtaPackageId newSoftwareId = device.getSoftwareId();
+        if (newSoftwareId == null) {
+            DeviceProfile newDeviceProfile = deviceProfileService.findDeviceProfileById(device.getTenantId(), device.getDeviceProfileId());
+            newSoftwareId = newDeviceProfile.getSoftwareId();
+        }
+        if (oldDevice != null) {
+            if (newSoftwareId != null) {
+                OtaPackageId oldSoftwareId = oldDevice.getSoftwareId();
+                if (oldSoftwareId == null) {
+                    DeviceProfile oldDeviceProfile = deviceProfileService.findDeviceProfileById(oldDevice.getTenantId(), oldDevice.getDeviceProfileId());
+                    oldSoftwareId = oldDeviceProfile.getSoftwareId();
+                }
+                if (!newSoftwareId.equals(oldSoftwareId)) {
+                    // Device was updated and new firmware is different from previous firmware.
+                    send(device.getTenantId(), device.getId(), newSoftwareId, System.currentTimeMillis(), SOFTWARE);
+                }
+            } else {
+                // Device was updated and new firmware is not set.
+                remove(device, SOFTWARE);
+            }
+        } else if (newSoftwareId != null) {
+            // Device was created and firmware is defined.
+            send(device.getTenantId(), device.getId(), newSoftwareId, System.currentTimeMillis(), SOFTWARE);
+        }
+    }
+
+    @Override
+    public void update(DeviceProfile deviceProfile, boolean isFirmwareChanged, boolean isSoftwareChanged) {
+        TenantId tenantId = deviceProfile.getTenantId();
+
+        if (isFirmwareChanged) {
+            update(tenantId, deviceProfile, FIRMWARE);
+        }
+        if (isSoftwareChanged) {
+            update(tenantId, deviceProfile, SOFTWARE);
+        }
+    }
+
+    private void update(TenantId tenantId, DeviceProfile deviceProfile, OtaPackageType otaPackageType) {
+        Consumer<Device> updateConsumer;
+
+        if (deviceProfile.getFirmwareId() != null) {
+            long ts = System.currentTimeMillis();
+            updateConsumer = d -> send(d.getTenantId(), d.getId(), deviceProfile.getFirmwareId(), ts, otaPackageType);
+        } else {
+            updateConsumer = d -> remove(d, otaPackageType);
+        }
+
+        PageLink pageLink = new PageLink(100);
+        PageData<Device> pageData;
+        do {
+            pageData = deviceService.findDevicesByTenantIdAndTypeAndEmptyOtaPackage(tenantId, deviceProfile.getId(), otaPackageType, pageLink);
+            pageData.getData().forEach(updateConsumer);
+
+            if (pageData.hasNext()) {
+                pageLink = pageLink.nextPageLink();
+            }
+        } while (pageData.hasNext());
+    }
+
+    @Override
+    public boolean process(ToOtaPackageStateServiceMsg msg) {
+        boolean isSuccess = false;
+        OtaPackageId targetOtaPackageId = new OtaPackageId(new UUID(msg.getOtaPackageIdMSB(), msg.getOtaPackageIdLSB()));
+        DeviceId deviceId = new DeviceId(new UUID(msg.getDeviceIdMSB(), msg.getDeviceIdLSB()));
+        TenantId tenantId = new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB()));
+        OtaPackageType firmwareType = OtaPackageType.valueOf(msg.getType());
+        long ts = msg.getTs();
+
+        Device device = deviceService.findDeviceById(tenantId, deviceId);
+        if (device == null) {
+            log.warn("[{}] [{}] Device was removed during firmware update msg was queued!", tenantId, deviceId);
+        } else {
+            OtaPackageId currentOtaPackageId = OtaPackageUtil.getOtaPackageId(device, firmwareType);
+            if (currentOtaPackageId == null) {
+                DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileById(tenantId, device.getDeviceProfileId());
+                currentOtaPackageId = OtaPackageUtil.getOtaPackageId(deviceProfile, firmwareType);
+            }
+
+            if (targetOtaPackageId.equals(currentOtaPackageId)) {
+                update(device, otaPackageService.findOtaPackageInfoById(device.getTenantId(), targetOtaPackageId), ts);
+                isSuccess = true;
+            } else {
+                log.warn("[{}] [{}] Can`t update firmware for the device, target firmwareId: [{}], current firmwareId: [{}]!", tenantId, deviceId, targetOtaPackageId, currentOtaPackageId);
+            }
+        }
+        return isSuccess;
+    }
+
+    private void send(TenantId tenantId, DeviceId deviceId, OtaPackageId firmwareId, long ts, OtaPackageType firmwareType) {
+        ToOtaPackageStateServiceMsg msg = ToOtaPackageStateServiceMsg.newBuilder()
+                .setTenantIdMSB(tenantId.getId().getMostSignificantBits())
+                .setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
+                .setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
+                .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
+                .setOtaPackageIdMSB(firmwareId.getId().getMostSignificantBits())
+                .setOtaPackageIdLSB(firmwareId.getId().getLeastSignificantBits())
+                .setType(firmwareType.name())
+                .setTs(ts)
+                .build();
+
+        OtaPackageInfo firmware = otaPackageService.findOtaPackageInfoById(tenantId, firmwareId);
+        if (firmware == null) {
+            log.warn("[{}] Failed to send firmware update because firmware was already deleted", firmwareId);
+            return;
+        }
+
+        TopicPartitionInfo tpi = new TopicPartitionInfo(otaPackageStateMsgProducer.getDefaultTopic(), null, null, false);
+        otaPackageStateMsgProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null);
+
+        List<TsKvEntry> telemetry = new ArrayList<>();
+        telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(getTargetTelemetryKey(firmware.getType(), TITLE), firmware.getTitle())));
+        telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(getTargetTelemetryKey(firmware.getType(), VERSION), firmware.getVersion())));
+        telemetry.add(new BasicTsKvEntry(ts, new LongDataEntry(getTargetTelemetryKey(firmware.getType(), TS), ts)));
+        telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(getTelemetryKey(firmware.getType(), STATE), OtaPackageUpdateStatus.QUEUED.name())));
+
+        telemetryService.saveAndNotify(tenantId, deviceId, telemetry, new FutureCallback<>() {
+            @Override
+            public void onSuccess(@Nullable Void tmp) {
+                log.trace("[{}] Success save firmware status!", deviceId);
+            }
+
+            @Override
+            public void onFailure(Throwable t) {
+                log.error("[{}] Failed to save firmware status!", deviceId, t);
+            }
+        });
+    }
+
+
+    private void update(Device device, OtaPackageInfo otaPackage, long ts) {
+        TenantId tenantId = device.getTenantId();
+        DeviceId deviceId = device.getId();
+        OtaPackageType otaPackageType = otaPackage.getType();
+
+        BasicTsKvEntry status = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(getTelemetryKey(otaPackageType, STATE), OtaPackageUpdateStatus.INITIATED.name()));
+
+        telemetryService.saveAndNotify(tenantId, deviceId, Collections.singletonList(status), new FutureCallback<>() {
+            @Override
+            public void onSuccess(@Nullable Void tmp) {
+                log.trace("[{}] Success save telemetry with target firmware for device!", deviceId);
+            }
+
+            @Override
+            public void onFailure(Throwable t) {
+                log.error("[{}] Failed to save telemetry with target firmware for device!", deviceId, t);
+            }
+        });
+
+        List<AttributeKvEntry> attributes = new ArrayList<>();
+        attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, TITLE), otaPackage.getTitle())));
+        attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, VERSION), otaPackage.getVersion())));
+        if (otaPackage.hasUrl()) {
+            attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, URL), otaPackage.getUrl())));
+            List<String> attrToRemove = new ArrayList<>();
+
+            if (otaPackage.getDataSize() == null) {
+                attrToRemove.add(getAttributeKey(otaPackageType, SIZE));
+            } else {
+                attributes.add(new BaseAttributeKvEntry(ts, new LongDataEntry(getAttributeKey(otaPackageType, SIZE), otaPackage.getDataSize())));
+            }
+
+            if (otaPackage.getChecksumAlgorithm() != null) {
+                attrToRemove.add(getAttributeKey(otaPackageType, CHECKSUM_ALGORITHM));
+            } else {
+                attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, CHECKSUM_ALGORITHM), otaPackage.getChecksumAlgorithm().name())));
+            }
+
+            if (StringUtils.isEmpty(otaPackage.getChecksum())) {
+                attrToRemove.add(getAttributeKey(otaPackageType, CHECKSUM));
+            } else {
+                attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, CHECKSUM), otaPackage.getChecksum())));
+            }
+
+            remove(device, otaPackageType, attrToRemove);
+        } else {
+            attributes.add(new BaseAttributeKvEntry(ts, new LongDataEntry(getAttributeKey(otaPackageType, SIZE), otaPackage.getDataSize())));
+            attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, CHECKSUM_ALGORITHM), otaPackage.getChecksumAlgorithm().name())));
+            attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, CHECKSUM), otaPackage.getChecksum())));
+            remove(device, otaPackageType, Collections.singletonList(getAttributeKey(otaPackageType, URL)));
+        }
+
+        telemetryService.saveAndNotify(tenantId, deviceId, DataConstants.SHARED_SCOPE, attributes, new FutureCallback<>() {
+            @Override
+            public void onSuccess(@Nullable Void tmp) {
+                log.trace("[{}] Success save attributes with target firmware!", deviceId);
+            }
+
+            @Override
+            public void onFailure(Throwable t) {
+                log.error("[{}] Failed to save attributes with target firmware!", deviceId, t);
+            }
+        });
+    }
+
+    private void remove(Device device, OtaPackageType otaPackageType) {
+        remove(device, otaPackageType, OtaPackageUtil.getAttributeKeys(otaPackageType));
+    }
+
+    private void remove(Device device, OtaPackageType otaPackageType, List<String> attributesKeys) {
+        telemetryService.deleteAndNotify(device.getTenantId(), device.getId(), DataConstants.SHARED_SCOPE, attributesKeys,
+                new FutureCallback<>() {
+                    @Override
+                    public void onSuccess(@Nullable Void tmp) {
+                        log.trace("[{}] Success remove target {} attributes!", device.getId(), otaPackageType);
+                        Set<AttributeKey> keysToNotify = new HashSet<>();
+                        attributesKeys.forEach(key -> keysToNotify.add(new AttributeKey(DataConstants.SHARED_SCOPE, key)));
+                        tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete(device.getTenantId(), device.getId(), keysToNotify), null);
+                    }
+
+                    @Override
+                    public void onFailure(Throwable t) {
+                        log.error("[{}] Failed to remove target {} attributes!", device.getId(), otaPackageType, t);
+                    }
+                });
+    }
+}

application/src/main/java/org/thingsboard/server/service/ota/OtaPackageStateService.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.ota;
+
+import org.thingsboard.server.common.data.Device;
+import org.thingsboard.server.common.data.DeviceProfile;
+import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
+
+public interface OtaPackageStateService {
+
+    void update(Device device, Device oldDevice);
+
+    void update(DeviceProfile deviceProfile, boolean isFirmwareChanged, boolean isSoftwareChanged);
+
+    boolean process(ToOtaPackageStateServiceMsg msg);
+
+}

application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java

@@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.HasName;
 import org.thingsboard.server.common.data.TbResource;
 import org.thingsboard.server.common.data.Tenant;
 import org.thingsboard.server.common.data.TenantProfile;
+import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.DeviceProfileId;
 import org.thingsboard.server.common.data.id.EdgeId;

application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@@ -24,6 +24,7 @@ import org.springframework.context.event.EventListener;
 import org.springframework.core.annotation.Order;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.common.util.ThingsBoardThreadFactory;
 import org.thingsboard.rule.engine.api.RpcError;
 import org.thingsboard.server.actors.ActorSystemContext;
@@ -35,7 +36,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
 import org.thingsboard.server.common.msg.queue.TbCallback;
 import org.thingsboard.server.common.stats.StatsFactory;
 import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
-import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
 import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto;
 import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto;
 import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto;
@@ -49,18 +50,20 @@ import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseP
 import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto;
 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
+import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
 import org.thingsboard.server.queue.TbQueueConsumer;
 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
-import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
+import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
 import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
 import org.thingsboard.server.service.edge.EdgeNotificationService;
+import org.thingsboard.server.service.ota.OtaPackageStateService;
 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
-import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
 import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
+import org.thingsboard.server.service.queue.processing.IdMsgPair;
 import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
 import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
 import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg;
@@ -97,6 +100,11 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
     @Value("${queue.core.stats.enabled:false}")
     private boolean statsEnabled;
 
+    @Value("${queue.core.ota.pack-interval-ms:60000}")
+    private long firmwarePackInterval;
+    @Value("${queue.core.ota.pack-size:100}")
+    private int firmwarePackSize;
+
     private final TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> mainConsumer;
     private final DeviceStateService stateService;
     private final TbApiUsageStateService statsService;
@@ -104,11 +112,15 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
     private final SubscriptionManagerService subscriptionManagerService;
     private final TbCoreDeviceRpcService tbCoreDeviceRpcService;
     private final EdgeNotificationService edgeNotificationService;
+    private final OtaPackageStateService firmwareStateService;
     private final TbCoreConsumerStats stats;
     protected final TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
+    private final TbQueueConsumer<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> firmwareStatesConsumer;
 
     protected volatile ExecutorService usageStatsExecutor;
 
+    private volatile ExecutorService firmwareStatesExecutor;
+
     public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory,
                                         ActorSystemContext actorContext,
                                         DeviceStateService stateService,
@@ -121,10 +133,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
                                         TbApiUsageStateService statsService,
                                         TbTenantProfileCache tenantProfileCache,
                                         TbApiUsageStateService apiUsageStateService,
-                                        EdgeNotificationService edgeNotificationService) {
+                                        EdgeNotificationService edgeNotificationService,
+                                        OtaPackageStateService firmwareStateService) {
         super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer());
         this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer();
         this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer();
+        this.firmwareStatesConsumer = tbCoreQueueFactory.createToOtaPackageStateServiceMsgConsumer();
         this.stateService = stateService;
         this.localSubscriptionService = localSubscriptionService;
         this.subscriptionManagerService = subscriptionManagerService;
@@ -132,12 +146,14 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
         this.edgeNotificationService = edgeNotificationService;
         this.stats = new TbCoreConsumerStats(statsFactory);
         this.statsService = statsService;
+        this.firmwareStateService = firmwareStateService;
     }
 
     @PostConstruct
     public void init() {
         super.init("tb-core-consumer", "tb-core-notifications-consumer");
-        this.usageStatsExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer"));
+        this.usageStatsExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer"));
+        this.firmwareStatesExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-firmware-notifications-consumer"));
     }
 
     @PreDestroy
@@ -146,6 +162,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
         if (usageStatsExecutor != null) {
             usageStatsExecutor.shutdownNow();
         }
+        if (firmwareStatesExecutor != null) {
+            firmwareStatesExecutor.shutdownNow();
+        }
     }
 
     @EventListener(ApplicationReadyEvent.class)
@@ -153,6 +172,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
     public void onApplicationEvent(ApplicationReadyEvent event) {
         super.onApplicationEvent(event);
         launchUsageStatsConsumer();
+        launchOtaPackageUpdateNotificationConsumer();
     }
 
     @Override
@@ -167,6 +187,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
                             .map(tpi -> tpi.newByTopic(usageStatsConsumer.getTopic()))
                             .collect(Collectors.toSet()));
         }
+        this.firmwareStatesConsumer.subscribe();
     }
 
     @Override
@@ -178,14 +199,17 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
                     if (msgs.isEmpty()) {
                         continue;
                     }
-                    ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = msgs.stream().collect(
-                            Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
+                    List<IdMsgPair<ToCoreMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).collect(Collectors.toList());
+                    ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = orderedMsgList.stream().collect(
+                            Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
                     CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
                     TbPackProcessingContext<TbProtoQueueMsg<ToCoreMsg>> ctx = new TbPackProcessingContext<>(
                             processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
                     PendingMsgHolder pendingMsgHolder = new PendingMsgHolder();
                     Future<?> packSubmitFuture = consumersExecutor.submit(() -> {
-                        pendingMap.forEach((id, msg) -> {
+                        orderedMsgList.forEach((element) -> {
+                            UUID id = element.getUuid();
+                            TbProtoQueueMsg<ToCoreMsg> msg = element.getMsg();
                             log.trace("[{}] Creating main callback for message: {}", id, msg.getValue());
                             TbCallback callback = new TbPackCallback<>(id, ctx);
                             try {
@@ -203,7 +227,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
                                 } else if (toCoreMsg.hasEdgeNotificationMsg()) {
                                     log.trace("[{}] Forwarding message to edge service {}", id, toCoreMsg.getEdgeNotificationMsg());
                                     forwardToEdgeNotificationService(toCoreMsg.getEdgeNotificationMsg(), callback);
-                                } else if (toCoreMsg.getToDeviceActorNotificationMsg() != null && !toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) {
+                                } else if (!toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) {
                                     Optional<TbActorMsg> actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray());
                                     if (actorMsg.isPresent()) {
                                         TbActorMsg tbActorMsg = actorMsg.get();
@@ -336,10 +360,59 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
         });
     }
 
+    private void launchOtaPackageUpdateNotificationConsumer() {
+        long maxProcessingTimeoutPerRecord = firmwarePackInterval / firmwarePackSize;
+        firmwareStatesExecutor.submit(() -> {
+            while (!stopped) {
+                try {
+                    List<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> msgs = firmwareStatesConsumer.poll(getNotificationPollDuration());
+                    if (msgs.isEmpty()) {
+                        continue;
+                    }
+                    long timeToSleep = maxProcessingTimeoutPerRecord;
+                    for (TbProtoQueueMsg<ToOtaPackageStateServiceMsg> msg : msgs) {
+                        try {
+                            long startTime = System.currentTimeMillis();
+                            boolean isSuccessUpdate = handleOtaPackageUpdates(msg);
+                            long endTime = System.currentTimeMillis();
+                            long spentTime = endTime - startTime;
+                            timeToSleep = timeToSleep - spentTime;
+                            if (isSuccessUpdate) {
+                                if (timeToSleep > 0) {
+                                    log.debug("Spent time per record is: [{}]!", spentTime);
+                                    Thread.sleep(timeToSleep);
+                                    timeToSleep = 0;
+                                }
+                                timeToSleep += maxProcessingTimeoutPerRecord;
+                            }
+                        } catch (Throwable e) {
+                            log.warn("Failed to process firmware update msg: {}", msg, e);
+                        }
+                    }
+                    firmwareStatesConsumer.commit();
+                } catch (Exception e) {
+                    if (!stopped) {
+                        log.warn("Failed to obtain usage stats from queue.", e);
+                        try {
+                            Thread.sleep(getNotificationPollDuration());
+                        } catch (InterruptedException e2) {
+                            log.trace("Failed to wait until the server has capacity to handle new firmware updates", e2);
+                        }
+                    }
+                }
+            }
+            log.info("TB Ota Package States Consumer stopped.");
+        });
+    }
+
     private void handleUsageStats(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
         statsService.process(msg, callback);
     }
 
+    private boolean handleOtaPackageUpdates(TbProtoQueueMsg<ToOtaPackageStateServiceMsg> msg) {
+        return firmwareStateService.process(msg.getValue());
+    }
+
     private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) {
         RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null;
         FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB())
@@ -448,6 +521,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
         if (usageStatsConsumer != null) {
             usageStatsConsumer.unsubscribe();
         }
+        if (firmwareStatesConsumer != null) {
+            firmwareStatesConsumer.unsubscribe();
+        }
     }
 
 }

application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.ThingsBoardThreadFactory;
 import org.thingsboard.rule.engine.api.RpcError;
 import org.thingsboard.server.actors.ActorSystemContext;
 import org.thingsboard.server.common.data.id.TenantId;
@@ -38,7 +39,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
 import org.thingsboard.server.queue.TbQueueConsumer;
 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
-import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
+import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
 import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
@@ -127,7 +128,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
             consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration));
             consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName(), statsFactory));
         }
-        submitExecutor = Executors.newSingleThreadExecutor();
+        submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-service-submit-executor"));
     }
 
     @PreDestroy
@@ -160,6 +161,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
 
     private void launchConsumer(TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats) {
         consumersExecutor.execute(() -> {
+            Thread.currentThread().setName("" + Thread.currentThread().getName() + "-" + configuration.getName());
             while (!stopped) {
                 try {
                     List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs = consumer.poll(pollDuration);

application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java

@@ -16,7 +16,7 @@
 package org.thingsboard.server.service.queue;
 
 import org.springframework.context.ApplicationListener;
-import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
+import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
 
 public interface TbCoreConsumerService extends ApplicationListener<PartitionChangeEvent> {
 

application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java

@@ -16,7 +16,7 @@
 package org.thingsboard.server.service.queue;
 
 import org.springframework.context.ApplicationListener;
-import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
+import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
 
 public interface TbRuleEngineConsumerService extends ApplicationListener<PartitionChangeEvent> {
 

application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@@ -18,14 +18,15 @@ package org.thingsboard.server.service.queue.processing;
 import com.google.protobuf.ByteString;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.context.event.ApplicationReadyEvent;
-import org.springframework.context.ApplicationListener;
 import org.springframework.context.event.EventListener;
 import org.springframework.core.annotation.Order;
 import org.thingsboard.common.util.ThingsBoardThreadFactory;
 import org.thingsboard.server.actors.ActorSystemContext;
 import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.DeviceProfileId;
+import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.id.TenantProfileId;
 import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
 import org.thingsboard.server.common.msg.TbActorMsg;
@@ -34,7 +35,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
 import org.thingsboard.server.common.msg.queue.TbCallback;
 import org.thingsboard.server.queue.TbQueueConsumer;
 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
-import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
+import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
 import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
 import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
 import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
@@ -72,7 +73,8 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
     protected final TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer;
 
     public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService,
-                                   TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache, TbApiUsageStateService apiUsageStateService, TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) {
+                                   TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache,
+                                   TbApiUsageStateService apiUsageStateService, TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) {
         this.actorContext = actorContext;
         this.encodingService = encodingService;
         this.tenantProfileCache = tenantProfileCache;
@@ -166,6 +168,8 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
                     tenantProfileCache.evict(componentLifecycleMsg.getTenantId());
                     if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) {
                         apiUsageStateService.onTenantUpdate(componentLifecycleMsg.getTenantId());
+                    } else if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.DELETED)) {
+                        apiUsageStateService.onTenantDelete((TenantId) componentLifecycleMsg.getEntityId());
                     }
                 } else if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
                     deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId()));
@@ -173,6 +177,10 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
                     deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceId(componentLifecycleMsg.getEntityId().getId()));
                 } else if (EntityType.API_USAGE_STATE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
                     apiUsageStateService.onApiUsageStateUpdate(componentLifecycleMsg.getTenantId());
+                } else if (EntityType.CUSTOMER.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
+                    if (componentLifecycleMsg.getEvent() == ComponentLifecycleEvent.DELETED) {
+                        apiUsageStateService.onCustomerDelete((CustomerId) componentLifecycleMsg.getEntityId());
+                    }
                 }
             }
             log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg);

application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java

@@ -27,7 +27,7 @@ import java.util.stream.Collectors;
 public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngineSubmitStrategy {
 
     protected final String queueName;
-    protected List<IdMsgPair> orderedMsgList;
+    protected List<IdMsgPair<TransportProtos.ToRuleEngineMsg>> orderedMsgList;
     private volatile boolean stopped;
 
     public AbstractTbRuleEngineSubmitStrategy(String queueName) {
@@ -38,7 +38,7 @@ public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngine
 
     @Override
     public void init(List<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> msgs) {
-        orderedMsgList = msgs.stream().map(msg -> new IdMsgPair(UUID.randomUUID(), msg)).collect(Collectors.toList());
+        orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).collect(Collectors.toList());
     }
 
     @Override
@@ -48,8 +48,8 @@ public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngine
 
     @Override
     public void update(ConcurrentMap<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> reprocessMap) {
-        List<IdMsgPair> newOrderedMsgList = new ArrayList<>(reprocessMap.size());
-        for (IdMsgPair pair : orderedMsgList) {
+        List<IdMsgPair<TransportProtos.ToRuleEngineMsg>> newOrderedMsgList = new ArrayList<>(reprocessMap.size());
+        for (IdMsgPair<TransportProtos.ToRuleEngineMsg> pair : orderedMsgList) {
             if (reprocessMap.containsKey(pair.uuid)) {
                 newOrderedMsgList.add(pair);
             }

application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java

@@ -15,16 +15,18 @@
  */
 package org.thingsboard.server.service.queue.processing;
 
-import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
+import lombok.Getter;
 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
 
 import java.util.UUID;
 
-public class IdMsgPair {
+public class IdMsgPair<T extends com.google.protobuf.GeneratedMessageV3> {
+    @Getter
     final UUID uuid;
-    final TbProtoQueueMsg<ToRuleEngineMsg> msg;
+    @Getter
+    final TbProtoQueueMsg<T> msg;
 
-    public IdMsgPair(UUID uuid, TbProtoQueueMsg<ToRuleEngineMsg> msg) {
+    public IdMsgPair(UUID uuid, TbProtoQueueMsg<T> msg) {
         this.uuid = uuid;
         this.msg = msg;
     }

application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java

@@ -33,7 +33,7 @@ public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends Abs
 
     private volatile BiConsumer<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> msgConsumer;
     private volatile ConcurrentMap<UUID, EntityId> msgToEntityIdMap = new ConcurrentHashMap<>();
-    private volatile ConcurrentMap<EntityId, Queue<IdMsgPair>> entityIdToListMap = new ConcurrentHashMap<>();
+    private volatile ConcurrentMap<EntityId, Queue<IdMsgPair<TransportProtos.ToRuleEngineMsg>>> entityIdToListMap = new ConcurrentHashMap<>();
 
     public SequentialByEntityIdTbRuleEngineSubmitStrategy(String queueName) {
         super(queueName);
@@ -66,7 +66,7 @@ public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends Abs
     protected void doOnSuccess(UUID id) {
         EntityId entityId = msgToEntityIdMap.get(id);
         if (entityId != null) {
-            Queue<IdMsgPair> queue = entityIdToListMap.get(entityId);
+            Queue<IdMsgPair<TransportProtos.ToRuleEngineMsg>> queue = entityIdToListMap.get(entityId);
             if (queue != null) {
                 IdMsgPair next = null;
                 synchronized (queue) {
@@ -86,7 +86,7 @@ public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends Abs
     private void initMaps() {
         msgToEntityIdMap.clear();
         entityIdToListMap.clear();
-        for (IdMsgPair pair : orderedMsgList) {
+        for (IdMsgPair<TransportProtos.ToRuleEngineMsg> pair : orderedMsgList) {
             EntityId entityId = getEntityId(pair.msg.getValue());
             if (entityId != null) {
                 msgToEntityIdMap.put(pair.uuid, entityId);

application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java

@@ -0,0 +1,218 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.resource;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.eclipse.leshan.core.model.DDFFileParser;
+import org.eclipse.leshan.core.model.DefaultDDFFileValidator;
+import org.eclipse.leshan.core.model.InvalidDDFFileException;
+import org.eclipse.leshan.core.model.ObjectModel;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.common.data.ResourceType;
+import org.thingsboard.server.common.data.TbResource;
+import org.thingsboard.server.common.data.TbResourceInfo;
+import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
+import org.thingsboard.server.common.data.exception.ThingsboardException;
+import org.thingsboard.server.common.data.id.TbResourceId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.lwm2m.LwM2mInstance;
+import org.thingsboard.server.common.data.lwm2m.LwM2mObject;
+import org.thingsboard.server.common.data.lwm2m.LwM2mResourceObserve;
+import org.thingsboard.server.common.data.page.PageData;
+import org.thingsboard.server.common.data.page.PageLink;
+import org.thingsboard.server.dao.exception.DataValidationException;
+import org.thingsboard.server.dao.resource.ResourceService;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_KEY;
+import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_SEARCH_TEXT;
+import static org.thingsboard.server.dao.device.DeviceServiceImpl.INCORRECT_TENANT_ID;
+import static org.thingsboard.server.dao.service.Validator.validateId;
+
+@Slf4j
+@Service
+public class DefaultTbResourceService implements TbResourceService {
+
+    private final ResourceService resourceService;
+    private final DDFFileParser ddfFileParser;
+
+    public DefaultTbResourceService(ResourceService resourceService) {
+        this.resourceService = resourceService;
+        this.ddfFileParser = new DDFFileParser(new DefaultDDFFileValidator());
+    }
+
+    @Override
+    public TbResource saveResource(TbResource resource) throws ThingsboardException {
+        log.trace("Executing saveResource [{}]", resource);
+        if (StringUtils.isEmpty(resource.getData())) {
+            throw new DataValidationException("Resource data should be specified!");
+        }
+        if (ResourceType.LWM2M_MODEL.equals(resource.getResourceType())) {
+            try {
+                List<ObjectModel> objectModels =
+                        ddfFileParser.parse(new ByteArrayInputStream(Base64.getDecoder().decode(resource.getData())), resource.getSearchText());
+                if (!objectModels.isEmpty()) {
+                    ObjectModel objectModel = objectModels.get(0);
+
+                    String resourceKey = objectModel.id + LWM2M_SEPARATOR_KEY + objectModel.version;
+                    String name = objectModel.name;
+                    resource.setResourceKey(resourceKey);
+                    if (resource.getId() == null) {
+                        resource.setTitle(name + " id=" + objectModel.id + " v" + objectModel.version);
+                    }
+                    resource.setSearchText(resourceKey + LWM2M_SEPARATOR_SEARCH_TEXT + name);
+                } else {
+                    throw new DataValidationException(String.format("Could not parse the XML of objectModel with name %s", resource.getSearchText()));
+                }
+            } catch (InvalidDDFFileException e) {
+                log.error("Failed to parse file {}", resource.getFileName(), e);
+                throw new DataValidationException("Failed to parse file " + resource.getFileName());
+            } catch (IOException e) {
+                throw new ThingsboardException(e, ThingsboardErrorCode.GENERAL);
+            }
+            if (resource.getResourceType().equals(ResourceType.LWM2M_MODEL) && toLwM2mObject(resource, true) == null) {
+                throw new DataValidationException(String.format("Could not parse the XML of objectModel with name %s", resource.getSearchText()));
+            }
+        } else {
+            resource.setResourceKey(resource.getFileName());
+        }
+
+        return resourceService.saveResource(resource);
+    }
+
+    @Override
+    public TbResource getResource(TenantId tenantId, ResourceType resourceType, String resourceId) {
+        return resourceService.getResource(tenantId, resourceType, resourceId);
+    }
+
+    @Override
+    public TbResource findResourceById(TenantId tenantId, TbResourceId resourceId) {
+        return resourceService.findResourceById(tenantId, resourceId);
+    }
+
+    @Override
+    public TbResourceInfo findResourceInfoById(TenantId tenantId, TbResourceId resourceId) {
+        return resourceService.findResourceInfoById(tenantId, resourceId);
+    }
+
+    @Override
+    public PageData<TbResourceInfo> findAllTenantResourcesByTenantId(TenantId tenantId, PageLink pageLink) {
+        return resourceService.findAllTenantResourcesByTenantId(tenantId, pageLink);
+    }
+
+    @Override
+    public PageData<TbResourceInfo> findTenantResourcesByTenantId(TenantId tenantId, PageLink pageLink) {
+        return resourceService.findTenantResourcesByTenantId(tenantId, pageLink);
+    }
+
+    @Override
+    public List<LwM2mObject> findLwM2mObject(TenantId tenantId, String sortOrder, String sortProperty, String[] objectIds) {
+        log.trace("Executing findByTenantId [{}]", tenantId);
+        validateId(tenantId, INCORRECT_TENANT_ID + tenantId);
+        List<TbResource> resources = resourceService.findTenantResourcesByResourceTypeAndObjectIds(tenantId, ResourceType.LWM2M_MODEL,
+                objectIds);
+        return resources.stream()
+                .flatMap(s -> Stream.ofNullable(toLwM2mObject(s, false)))
+                .sorted(getComparator(sortProperty, sortOrder))
+                .collect(Collectors.toList());
+    }
+
+    @Override
+    public List<LwM2mObject> findLwM2mObjectPage(TenantId tenantId, String sortProperty, String sortOrder, PageLink pageLink) {
+        log.trace("Executing findByTenantId [{}]", tenantId);
+        validateId(tenantId, INCORRECT_TENANT_ID + tenantId);
+        PageData<TbResource> resourcePageData = resourceService.findTenantResourcesByResourceTypeAndPageLink(tenantId, ResourceType.LWM2M_MODEL, pageLink);
+        return resourcePageData.getData().stream()
+                .flatMap(s -> Stream.ofNullable(toLwM2mObject(s, false)))
+                .sorted(getComparator(sortProperty, sortOrder))
+                .collect(Collectors.toList());
+    }
+
+    @Override
+    public void deleteResource(TenantId tenantId, TbResourceId resourceId) {
+        resourceService.deleteResource(tenantId, resourceId);
+    }
+
+    @Override
+    public void deleteResourcesByTenantId(TenantId tenantId) {
+        resourceService.deleteResourcesByTenantId(tenantId);
+    }
+
+    @Override
+    public long sumDataSizeByTenantId(TenantId tenantId) {
+        return resourceService.sumDataSizeByTenantId(tenantId);
+    }
+
+    private Comparator<? super LwM2mObject> getComparator(String sortProperty, String sortOrder) {
+        Comparator<LwM2mObject> comparator;
+        if ("name".equals(sortProperty)) {
+            comparator = Comparator.comparing(LwM2mObject::getName);
+        } else {
+            comparator = Comparator.comparingLong(LwM2mObject::getId);
+        }
+        return "DESC".equals(sortOrder) ? comparator.reversed() : comparator;
+    }
+
+    private LwM2mObject toLwM2mObject(TbResource resource, boolean isSave) {
+        try {
+            DDFFileParser ddfFileParser = new DDFFileParser(new DefaultDDFFileValidator());
+            List<ObjectModel> objectModels =
+                    ddfFileParser.parse(new ByteArrayInputStream(Base64.getDecoder().decode(resource.getData())), resource.getSearchText());
+            if (objectModels.size() == 0) {
+                return null;
+            } else {
+                ObjectModel obj = objectModels.get(0);
+                LwM2mObject lwM2mObject = new LwM2mObject();
+                lwM2mObject.setId(obj.id);
+                lwM2mObject.setKeyId(resource.getResourceKey());
+                lwM2mObject.setName(obj.name);
+                lwM2mObject.setMultiple(obj.multiple);
+                lwM2mObject.setMandatory(obj.mandatory);
+                LwM2mInstance instance = new LwM2mInstance();
+                instance.setId(0);
+                List<LwM2mResourceObserve> resources = new ArrayList<>();
+                obj.resources.forEach((k, v) -> {
+                    if (isSave) {
+                        LwM2mResourceObserve lwM2MResourceObserve = new LwM2mResourceObserve(k, v.name, false, false, false);
+                        resources.add(lwM2MResourceObserve);
+                    } else if (v.operations.isReadable()) {
+                        LwM2mResourceObserve lwM2MResourceObserve = new LwM2mResourceObserve(k, v.name, false, false, false);
+                        resources.add(lwM2MResourceObserve);
+                    }
+                });
+                if (isSave || resources.size() > 0) {
+                    instance.setResources(resources.toArray(LwM2mResourceObserve[]::new));
+                    lwM2mObject.setInstances(new LwM2mInstance[]{instance});
+                    return lwM2mObject;
+                } else {
+                    return null;
+                }
+            }
+        } catch (IOException | InvalidDDFFileException e) {
+            log.error("Could not parse the XML of objectModel with name [{}]", resource.getSearchText(), e);
+            return null;
+        }
+    }
+}

common/dao-api/src/main/java/org/thingsboard/server/dao/resource/TbResourceService.java → application/src/main/java/org/thingsboard/server/service/resource/TbResourceService.java

@@ -13,26 +13,25 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.thingsboard.server.dao.resource;
+package org.thingsboard.server.service.resource;
 
-import org.eclipse.leshan.core.model.InvalidDDFFileException;
+import org.thingsboard.server.common.data.ResourceType;
 import org.thingsboard.server.common.data.TbResource;
 import org.thingsboard.server.common.data.TbResourceInfo;
-import org.thingsboard.server.common.data.ResourceType;
+import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.id.TbResourceId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.lwm2m.LwM2mObject;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.page.PageLink;
 
-import java.io.IOException;
 import java.util.List;
 
-
 public interface TbResourceService {
-    TbResource saveResource(TbResource resource) throws InvalidDDFFileException, IOException;
 
-    TbResource getResource(TenantId tenantId, ResourceType resourceType, String resourceId);
+    TbResource saveResource(TbResource resource) throws ThingsboardException;
+
+    TbResource getResource(TenantId tenantId, ResourceType resourceType, String resourceKey);
 
     TbResource findResourceById(TenantId tenantId, TbResourceId resourceId);
 
@@ -55,4 +54,6 @@ public interface TbResourceService {
     void deleteResource(TenantId tenantId, TbResourceId resourceId);
 
     void deleteResourcesByTenantId(TenantId tenantId);
+
+    long sumDataSizeByTenantId(TenantId tenantId);
 }

application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java

@@ -34,6 +34,7 @@ import org.thingsboard.server.dao.device.DeviceService;
 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.queue.TbClusterService;
+import org.thingsboard.server.service.security.model.SecurityUser;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
@@ -95,11 +96,11 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
     }
 
     @Override
-    public void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer) {
+    public void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer, SecurityUser currentUser) {
         log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId());
         UUID requestId = request.getId();
         localToRuleEngineRpcRequests.put(requestId, responseConsumer);
-        sendRpcRequestToRuleEngine(request);
+        sendRpcRequestToRuleEngine(request, currentUser);
         scheduleToRuleEngineTimeout(request, requestId);
     }
 
@@ -149,13 +150,14 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
         }
     }
 
-    private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) {
+    private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg, SecurityUser currentUser) {
         ObjectNode entityNode = json.createObjectNode();
         TbMsgMetaData metaData = new TbMsgMetaData();
         metaData.putValue("requestUUID", msg.getId().toString());
         metaData.putValue("originServiceId", serviceId);
         metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime()));
         metaData.putValue("oneway", Boolean.toString(msg.isOneway()));
+        metaData.putValue("persisted", Boolean.toString(msg.isPersisted()));
 
         Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId());
         if (device != null) {
@@ -167,7 +169,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
         entityNode.put("params", msg.getBody().getParams());
 
         try {
-            TbMsg tbMsg = TbMsg.newMsg(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode));
+            TbMsg tbMsg = TbMsg.newMsg(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), currentUser.getCustomerId(), metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode));
             clusterService.pushMsgToRuleEngine(msg.getTenantId(), msg.getDeviceId(), tbMsg, null);
         } catch (JsonProcessingException e) {
             throw new RuntimeException(e);

application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java

@@ -100,7 +100,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi
     @Override
     public void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest src, Consumer<RuleEngineDeviceRpcResponse> consumer) {
         ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), src.getTenantId(), src.getDeviceId(),
-                src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody()));
+                src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody()), src.isPersisted());
         forwardRpcRequestToDeviceActor(request, response -> {
             if (src.isRestApiCall()) {
                 sendRpcResponseToTbCore(src.getOriginServiceId(), response);

application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.service.rpc;
 
 import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest;
+import org.thingsboard.server.service.security.model.SecurityUser;
 
 import java.util.function.Consumer;
 
@@ -27,11 +28,11 @@ public interface TbCoreDeviceRpcService {
     /**
      * Handles REST API calls that contain RPC requests to Device and pushes them to Rule Engine.
      * Schedules the timeout for the RPC call based on the {@link ToDeviceRpcRequest}
-     *
-     * @param request          the RPC request
+     *  @param request          the RPC request
      * @param responseConsumer the consumer of the RPC response
+     * @param currentUser
      */
-    void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer);
+    void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer, SecurityUser currentUser);
 
     /**
      * Handles the RPC response from the Rule Engine.

application/src/main/java/org/thingsboard/server/service/rpc/TbRpcService.java

@@ -0,0 +1,77 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.rpc;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.server.common.data.id.DeviceId;
+import org.thingsboard.server.common.data.id.RpcId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.page.PageData;
+import org.thingsboard.server.common.data.page.PageLink;
+import org.thingsboard.server.common.data.rpc.Rpc;
+import org.thingsboard.server.common.data.rpc.RpcStatus;
+import org.thingsboard.server.common.msg.TbMsg;
+import org.thingsboard.server.common.msg.TbMsgMetaData;
+import org.thingsboard.server.dao.rpc.RpcService;
+import org.thingsboard.server.queue.util.TbCoreComponent;
+import org.thingsboard.server.service.queue.TbClusterService;
+
+@TbCoreComponent
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class TbRpcService {
+    private final RpcService rpcService;
+    private final TbClusterService tbClusterService;
+
+    public Rpc save(TenantId tenantId, Rpc rpc) {
+        Rpc saved = rpcService.save(rpc);
+        pushRpcMsgToRuleEngine(tenantId, saved);
+        return saved;
+    }
+
+    public void save(TenantId tenantId, RpcId rpcId, RpcStatus newStatus, JsonNode response) {
+        Rpc foundRpc = rpcService.findById(tenantId, rpcId);
+        if (foundRpc != null) {
+            foundRpc.setStatus(newStatus);
+            if (response != null) {
+                foundRpc.setResponse(response);
+            }
+            Rpc saved = rpcService.save(foundRpc);
+            pushRpcMsgToRuleEngine(tenantId, saved);
+        } else {
+            log.warn("[{}] Failed to update RPC status because RPC was already deleted", rpcId);
+        }
+    }
+
+    private void pushRpcMsgToRuleEngine(TenantId tenantId, Rpc rpc) {
+        TbMsg msg = TbMsg.newMsg("RPC_" + rpc.getStatus().name(), rpc.getDeviceId(), TbMsgMetaData.EMPTY, JacksonUtil.toString(rpc));
+        tbClusterService.pushMsgToRuleEngine(tenantId, rpc.getId(), msg, null);
+    }
+
+    public Rpc findRpcById(TenantId tenantId, RpcId rpcId) {
+        return rpcService.findById(tenantId, rpcId);
+    }
+
+    public PageData<Rpc> findAllByDeviceIdAndStatus(TenantId tenantId, DeviceId deviceId, RpcStatus rpcStatus, PageLink pageLink) {
+        return rpcService.findAllByDeviceIdAndStatus(tenantId, deviceId, rpcStatus, pageLink);
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java

@@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture;
 import lombok.extern.slf4j.Slf4j;
 import org.thingsboard.common.util.ThingsBoardThreadFactory;
 import org.thingsboard.server.common.data.ApiUsageRecordKey;
+import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
 import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
@@ -29,6 +30,7 @@ import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 
 /**
@@ -73,18 +75,20 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
     }
 
     @Override
-    public ListenableFuture<Object> invokeFunction(TenantId tenantId, UUID scriptId, Object... args) {
+    public ListenableFuture<Object> invokeFunction(TenantId tenantId, CustomerId customerId, UUID scriptId, Object... args) {
         if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) {
             String functionName = scriptIdToNameMap.get(scriptId);
             if (functionName == null) {
                 return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!"));
             }
             if (!isDisabled(scriptId)) {
-                apiUsageClient.report(tenantId, ApiUsageRecordKey.JS_EXEC_COUNT, 1);
+                apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.JS_EXEC_COUNT, 1);
                 return doInvokeFunction(scriptId, functionName, args);
             } else {
-                return Futures.immediateFailedFuture(
-                        new RuntimeException("Script invocation is blocked due to maximum error count " + getMaxErrors() + "!"));
+                String message = "Script invocation is blocked due to maximum error count "
+                        + getMaxErrors() + ", scriptId " + scriptId + "!";
+                log.warn(message);
+                return Futures.immediateFailedFuture(new RuntimeException(message));
             }
         } else {
             return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!"));
@@ -116,8 +120,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
 
     protected abstract long getMaxBlacklistDuration();
 
-    protected void onScriptExecutionError(UUID scriptId) {
-        disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).incrementAndGet();
+    protected void onScriptExecutionError(UUID scriptId, Throwable t, String scriptBody) {
+        DisableListInfo disableListInfo = disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo());
+        log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}, scriptBody {}",
+                disableListInfo.get(), scriptId, t, t.getCause(), scriptBody);
+        disableListInfo.incrementAndGet();
     }
 
     private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) {

application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java

@@ -25,6 +25,7 @@ import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.scheduling.annotation.Scheduled;
+import org.thingsboard.common.util.ThingsBoardExecutors;
 import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
 import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
 
@@ -93,7 +94,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
         super.init(maxRequestsTimeout);
         if (useJsSandbox()) {
             sandbox = NashornSandboxes.create();
-            monitorExecutorService = Executors.newWorkStealingPool(getMonitorThreadPoolSize());
+            monitorExecutorService = ThingsBoardExecutors.newWorkStealingPool(getMonitorThreadPoolSize(), "nashorn-js-monitor");
             sandbox.setExecutor(monitorExecutorService);
             sandbox.setMaxCPUTime(getMaxCpuTime());
             sandbox.allowNoBraces(false);
@@ -159,7 +160,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
                     return ((Invocable) engine).invokeFunction(functionName, args);
                 }
             } catch (Exception e) {
-                onScriptExecutionError(scriptId);
+                onScriptExecutionError(scriptId, e, functionName);
                 throw new ExecutionException(e);
             }
         });

application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java

@@ -16,7 +16,7 @@
 package org.thingsboard.server.service.script;
 
 import com.google.common.util.concurrent.ListenableFuture;
-import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.TenantId;
 
 import java.util.UUID;
@@ -25,7 +25,7 @@ public interface JsInvokeService {
 
     ListenableFuture<UUID> eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames);
 
-    ListenableFuture<Object> invokeFunction(TenantId tenantId, UUID scriptId, Object... args);
+    ListenableFuture<Object> invokeFunction(TenantId tenantId, CustomerId customerId, UUID scriptId, Object... args);
 
     ListenableFuture<Void> release(UUID scriptId);
 

application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java

@@ -18,7 +18,6 @@ package org.thingsboard.server.service.script;
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.MoreExecutors;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -26,6 +25,8 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
+import org.springframework.util.StopWatch;
+import org.thingsboard.common.util.ThingsBoardThreadFactory;
 import org.thingsboard.server.gen.js.JsInvokeProtos;
 import org.thingsboard.server.queue.TbQueueRequestTemplate;
 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
@@ -39,6 +40,8 @@ import javax.annotation.PreDestroy;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -69,6 +72,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
     private final AtomicInteger queueEvalMsgs = new AtomicInteger(0);
     private final AtomicInteger queueFailedMsgs = new AtomicInteger(0);
     private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0);
+    private final ExecutorService callbackExecutor = Executors.newFixedThreadPool(
+            Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("js-executor-remote-callback"));
 
     public RemoteJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) {
         super(apiUsageStateService, apiUsageClient);
@@ -139,7 +144,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
                 }
                 queueFailedMsgs.incrementAndGet();
             }
-        }, MoreExecutors.directExecutor());
+        }, callbackExecutor);
         return Futures.transform(future, response -> {
             JsInvokeProtos.JsCompileResponse compilationResult = response.getValue().getCompileResponse();
             UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB());
@@ -151,12 +156,13 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
                 log.debug("[{}] Failed to compile script due to [{}]: {}", compiledScriptId, compilationResult.getErrorCode().name(), compilationResult.getErrorDetails());
                 throw new RuntimeException(compilationResult.getErrorDetails());
             }
-        }, MoreExecutors.directExecutor());
+        }, callbackExecutor);
     }
 
     @Override
     protected ListenableFuture<Object> doInvokeFunction(UUID scriptId, String functionName, Object[] args) {
-        String scriptBody = scriptIdToBodysMap.get(scriptId);
+        log.trace("doInvokeFunction js-request for uuid {} with timeout {}ms", scriptId, maxRequestsTimeout);
+        final String scriptBody = scriptIdToBodysMap.get(scriptId);
         if (scriptBody == null) {
             return Futures.immediateFailedFuture(new RuntimeException("No script body found for scriptId: [" + scriptId + "]!"));
         }
@@ -165,7 +171,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
                 .setScriptIdLSB(scriptId.getLeastSignificantBits())
                 .setFunctionName(functionName)
                 .setTimeout((int) maxRequestsTimeout)
-                .setScriptBody(scriptIdToBodysMap.get(scriptId));
+                .setScriptBody(scriptBody);
 
         for (Object arg : args) {
             jsRequestBuilder.addArgs(arg.toString());
@@ -175,6 +181,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
                 .setInvokeRequest(jsRequestBuilder.build())
                 .build();
 
+        StopWatch stopWatch = new StopWatch();
+        stopWatch.start();
+
         ListenableFuture<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper));
         if (maxRequestsTimeout > 0) {
             future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
@@ -188,23 +197,26 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
 
             @Override
             public void onFailure(Throwable t) {
-                onScriptExecutionError(scriptId);
+                onScriptExecutionError(scriptId, t, scriptBody);
                 if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) {
                     queueTimeoutMsgs.incrementAndGet();
                 }
                 queueFailedMsgs.incrementAndGet();
             }
-        }, MoreExecutors.directExecutor());
+        }, callbackExecutor);
         return Futures.transform(future, response -> {
+            stopWatch.stop();
+            log.trace("doInvokeFunction js-response took {}ms for uuid {}", stopWatch.getTotalTimeMillis(), response.getKey());
             JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse();
             if (invokeResult.getSuccess()) {
                 return invokeResult.getResult();
             } else {
-                onScriptExecutionError(scriptId);
+                final RuntimeException e = new RuntimeException(invokeResult.getErrorDetails());
+                onScriptExecutionError(scriptId, e, scriptBody);
                 log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails());
-                throw new RuntimeException(invokeResult.getErrorDetails());
+                throw e;
             }
-        }, MoreExecutors.directExecutor());
+        }, callbackExecutor);
     }
 
     @Override

application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java

@@ -18,12 +18,12 @@ package org.thingsboard.server.service.script;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.MoreExecutors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.msg.TbMsg;
@@ -32,6 +32,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData;
 import javax.script.ScriptException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -102,140 +103,115 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
             String newMessageType = !StringUtils.isEmpty(messageType) ? messageType : msg.getType();
             return TbMsg.transformMsg(msg, newMessageType, msg.getOriginator(), newMetadata, newData);
         } catch (Throwable th) {
-            th.printStackTrace();
             throw new RuntimeException("Failed to unbind message data from javascript result", th);
         }
     }
 
     @Override
-    public List<TbMsg> executeUpdate(TbMsg msg) throws ScriptException {
-        JsonNode result = executeScript(msg);
-        if (result.isObject()) {
-            return Collections.singletonList(unbindMsg(result, msg));
-        } else if (result.isArray()){
-            List<TbMsg> res = new ArrayList<>(result.size());
-            result.forEach(jsonObject -> res.add(unbindMsg(jsonObject, msg)));
-            return res;
-        } else {
-            log.warn("Wrong result type: {}", result.getNodeType());
-            throw new ScriptException("Wrong result type: " + result.getNodeType());
+    public ListenableFuture<List<TbMsg>> executeUpdateAsync(TbMsg msg) {
+        ListenableFuture<JsonNode> result = executeScriptAsync(msg);
+        return Futures.transformAsync(result,
+                json -> executeUpdateTransform(msg, json),
+                MoreExecutors.directExecutor());
+    }
+
+    ListenableFuture<List<TbMsg>> executeUpdateTransform(TbMsg msg, JsonNode json) {
+        if (json.isObject()) {
+            return Futures.immediateFuture(Collections.singletonList(unbindMsg(json, msg)));
+        } else if (json.isArray()) {
+            List<TbMsg> res = new ArrayList<>(json.size());
+            json.forEach(jsonObject -> res.add(unbindMsg(jsonObject, msg)));
+            return Futures.immediateFuture(res);
         }
+        log.warn("Wrong result type: {}", json.getNodeType());
+        return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType()));
     }
 
     @Override
-    public ListenableFuture<List<TbMsg>> executeUpdateAsync(TbMsg msg) {
-        ListenableFuture<JsonNode> result = executeScriptAsync(msg);
-        return Futures.transformAsync(result, json -> {
-            if (json.isObject()) {
-                return Futures.immediateFuture(Collections.singletonList(unbindMsg(json, msg)));
-            } else if (json.isArray()){
-                List<TbMsg> res = new ArrayList<>(json.size());
-                json.forEach(jsonObject -> res.add(unbindMsg(jsonObject, msg)));
-                return Futures.immediateFuture(res);
-            }
-            else{
-                log.warn("Wrong result type: {}", json.getNodeType());
-                return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType()));
-            }
-        }, MoreExecutors.directExecutor());
+    public ListenableFuture<TbMsg> executeGenerateAsync(TbMsg prevMsg) {
+        return Futures.transformAsync(executeScriptAsync(prevMsg),
+                result -> executeGenerateTransform(prevMsg, result),
+                MoreExecutors.directExecutor());
     }
 
-    @Override
-    public TbMsg executeGenerate(TbMsg prevMsg) throws ScriptException {
-        JsonNode result = executeScript(prevMsg);
+    ListenableFuture<TbMsg> executeGenerateTransform(TbMsg prevMsg, JsonNode result) {
         if (!result.isObject()) {
             log.warn("Wrong result type: {}", result.getNodeType());
-            throw new ScriptException("Wrong result type: " + result.getNodeType());
+            Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType()));
         }
-        return unbindMsg(result, prevMsg);
+        return Futures.immediateFuture(unbindMsg(result, prevMsg));
     }
 
     @Override
-    public JsonNode executeJson(TbMsg msg) throws ScriptException {
-        return executeScript(msg);
-    }
-
-    @Override
-    public ListenableFuture<JsonNode> executeJsonAsync(TbMsg msg) throws ScriptException {
+    public ListenableFuture<JsonNode> executeJsonAsync(TbMsg msg) {
         return executeScriptAsync(msg);
     }
 
     @Override
-    public String executeToString(TbMsg msg) throws ScriptException {
-        JsonNode result = executeScript(msg);
-        if (!result.isTextual()) {
-            log.warn("Wrong result type: {}", result.getNodeType());
-            throw new ScriptException("Wrong result type: " + result.getNodeType());
-        }
-        return result.asText();
+    public ListenableFuture<String> executeToStringAsync(TbMsg msg) {
+        return Futures.transformAsync(executeScriptAsync(msg),
+                this::executeToStringTransform,
+                MoreExecutors.directExecutor());
     }
 
-    @Override
-    public boolean executeFilter(TbMsg msg) throws ScriptException {
-        JsonNode result = executeScript(msg);
-        if (!result.isBoolean()) {
-            log.warn("Wrong result type: {}", result.getNodeType());
-            throw new ScriptException("Wrong result type: " + result.getNodeType());
+    ListenableFuture<String> executeToStringTransform(JsonNode result) {
+        if (result.isTextual()) {
+            return Futures.immediateFuture(result.asText());
         }
-        return result.asBoolean();
+        log.warn("Wrong result type: {}", result.getNodeType());
+        return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType()));
     }
 
     @Override
     public ListenableFuture<Boolean> executeFilterAsync(TbMsg msg) {
-        ListenableFuture<JsonNode> result = executeScriptAsync(msg);
-        return Futures.transformAsync(result, json -> {
-            if (!json.isBoolean()) {
-                log.warn("Wrong result type: {}", json.getNodeType());
-                return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType()));
-            } else {
-                return Futures.immediateFuture(json.asBoolean());
-            }
-        }, MoreExecutors.directExecutor());
+        return Futures.transformAsync(executeScriptAsync(msg),
+                this::executeFilterTransform,
+                MoreExecutors.directExecutor());
     }
 
-    @Override
-    public Set<String> executeSwitch(TbMsg msg) throws ScriptException {
-        JsonNode result = executeScript(msg);
+    ListenableFuture<Boolean> executeFilterTransform(JsonNode json) {
+        if (json.isBoolean()) {
+            return Futures.immediateFuture(json.asBoolean());
+        }
+        log.warn("Wrong result type: {}", json.getNodeType());
+        return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType()));
+    }
+
+    ListenableFuture<Set<String>> executeSwitchTransform(JsonNode result) {
         if (result.isTextual()) {
-            return Collections.singleton(result.asText());
-        } else if (result.isArray()) {
-            Set<String> nextStates = Sets.newHashSet();
+            return Futures.immediateFuture(Collections.singleton(result.asText()));
+        }
+        if (result.isArray()) {
+            Set<String> nextStates = new HashSet<>();
             for (JsonNode val : result) {
                 if (!val.isTextual()) {
                     log.warn("Wrong result type: {}", val.getNodeType());
-                    throw new ScriptException("Wrong result type: " + val.getNodeType());
+                    return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + val.getNodeType()));
                 } else {
                     nextStates.add(val.asText());
                 }
             }
-            return nextStates;
-        } else {
-            log.warn("Wrong result type: {}", result.getNodeType());
-            throw new ScriptException("Wrong result type: " + result.getNodeType());
+            return Futures.immediateFuture(nextStates);
         }
+        log.warn("Wrong result type: {}", result.getNodeType());
+        return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType()));
     }
 
-    private JsonNode executeScript(TbMsg msg) throws ScriptException {
-        try {
-            String[] inArgs = prepareArgs(msg);
-            String eval = sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString();
-            return mapper.readTree(eval);
-        } catch (ExecutionException e) {
-            if (e.getCause() instanceof ScriptException) {
-                throw (ScriptException) e.getCause();
-            } else if (e.getCause() instanceof RuntimeException) {
-                throw new ScriptException(e.getCause().getMessage());
-            } else {
-                throw new ScriptException(e);
-            }
-        } catch (Exception e) {
-            throw new ScriptException(e);
-        }
+    @Override
+    public ListenableFuture<Set<String>> executeSwitchAsync(TbMsg msg) {
+        return Futures.transformAsync(executeScriptAsync(msg),
+                this::executeSwitchTransform,
+                MoreExecutors.directExecutor()); //usually runs in a callbackExecutor
     }
 
-    private ListenableFuture<JsonNode> executeScriptAsync(TbMsg msg) {
+    ListenableFuture<JsonNode> executeScriptAsync(TbMsg msg) {
+        log.trace("execute script async, msg {}", msg);
         String[] inArgs = prepareArgs(msg);
-        return Futures.transformAsync(sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]),
+        return executeScriptAsync(msg.getCustomerId(), inArgs[0], inArgs[1], inArgs[2]);
+    }
+
+    ListenableFuture<JsonNode> executeScriptAsync(CustomerId customerId, Object... args) {
+        return Futures.transformAsync(sandboxService.invokeFunction(tenantId, customerId, this.scriptId, args),
                 o -> {
                     try {
                         return Futures.immediateFuture(mapper.readTree(o.toString()));

application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java

@@ -30,6 +30,8 @@ import org.thingsboard.server.common.data.Customer;
 import org.thingsboard.server.common.data.Device;
 import org.thingsboard.server.common.data.DeviceProfile;
 import org.thingsboard.server.common.data.EntityView;
+import org.thingsboard.server.common.data.OtaPackageInfo;
+import org.thingsboard.server.common.data.TbResourceInfo;
 import org.thingsboard.server.common.data.Tenant;
 import org.thingsboard.server.common.data.User;
 import org.thingsboard.server.common.data.asset.Asset;
@@ -44,10 +46,14 @@ import org.thingsboard.server.common.data.id.EdgeId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.EntityIdFactory;
 import org.thingsboard.server.common.data.id.EntityViewId;
+import org.thingsboard.server.common.data.id.OtaPackageId;
+import org.thingsboard.server.common.data.id.RpcId;
 import org.thingsboard.server.common.data.id.RuleChainId;
 import org.thingsboard.server.common.data.id.RuleNodeId;
+import org.thingsboard.server.common.data.id.TbResourceId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.id.UserId;
+import org.thingsboard.server.common.data.rpc.Rpc;
 import org.thingsboard.server.common.data.rule.RuleChain;
 import org.thingsboard.server.common.data.rule.RuleNode;
 import org.thingsboard.server.controller.HttpValidationCallback;
@@ -59,6 +65,9 @@ import org.thingsboard.server.dao.device.DeviceService;
 import org.thingsboard.server.dao.edge.EdgeService;
 import org.thingsboard.server.dao.entityview.EntityViewService;
 import org.thingsboard.server.dao.exception.IncorrectParameterException;
+import org.thingsboard.server.dao.ota.OtaPackageService;
+import org.thingsboard.server.dao.resource.ResourceService;
+import org.thingsboard.server.dao.rpc.RpcService;
 import org.thingsboard.server.dao.rule.RuleChainService;
 import org.thingsboard.server.dao.tenant.TenantService;
 import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
@@ -125,6 +134,15 @@ public class AccessValidator {
     @Autowired
     protected ApiUsageStateService apiUsageStateService;
 
+    @Autowired
+    protected ResourceService resourceService;
+
+    @Autowired
+    protected OtaPackageService otaPackageService;
+
+    @Autowired
+    protected RpcService rpcService;
+
     private ExecutorService executor;
 
     @PostConstruct
@@ -217,6 +235,15 @@ public class AccessValidator {
             case API_USAGE_STATE:
                 validateApiUsageState(currentUser, operation, entityId, callback);
                 return;
+            case TB_RESOURCE:
+                validateResource(currentUser, operation, entityId, callback);
+                return;
+            case OTA_PACKAGE:
+                validateOtaPackage(currentUser, operation, entityId, callback);
+                return;
+            case RPC:
+                validateRpc(currentUser, operation, entityId, callback);
+                return;
             default:
                 //TODO: add support of other entities
                 throw new IllegalStateException("Not Implemented!");
@@ -243,6 +270,22 @@ public class AccessValidator {
         }
     }
 
+    private void validateRpc(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
+        ListenableFuture<Rpc> rpcFurure = rpcService.findRpcByIdAsync(currentUser.getTenantId(), new RpcId(entityId.getId()));
+        Futures.addCallback(rpcFurure, getCallback(callback, rpc -> {
+            if (rpc == null) {
+                return ValidationResult.entityNotFound("Rpc with requested id wasn't found!");
+            } else {
+                try {
+                    accessControlService.checkPermission(currentUser, Resource.RPC, operation, entityId, rpc);
+                } catch (ThingsboardException e) {
+                    return ValidationResult.accessDenied(e.getMessage());
+                }
+                return ValidationResult.ok(rpc);
+            }
+        }), executor);
+    }
+
     private void validateDeviceProfile(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
         if (currentUser.isSystemAdmin()) {
             callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION));
@@ -282,6 +325,40 @@ public class AccessValidator {
         }
     }
 
+    private void validateOtaPackage(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
+        if (currentUser.isSystemAdmin()) {
+            callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION));
+        } else {
+            OtaPackageInfo otaPackage = otaPackageService.findOtaPackageInfoById(currentUser.getTenantId(), new OtaPackageId(entityId.getId()));
+            if (otaPackage == null) {
+                callback.onSuccess(ValidationResult.entityNotFound("OtaPackage with requested id wasn't found!"));
+            } else {
+                try {
+                    accessControlService.checkPermission(currentUser, Resource.OTA_PACKAGE, operation, entityId, otaPackage);
+                } catch (ThingsboardException e) {
+                    callback.onSuccess(ValidationResult.accessDenied(e.getMessage()));
+                }
+                callback.onSuccess(ValidationResult.ok(otaPackage));
+            }
+        }
+    }
+
+    private void validateResource(SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
+        ListenableFuture<TbResourceInfo> resourceFuture = resourceService.findResourceInfoByIdAsync(currentUser.getTenantId(), new TbResourceId(entityId.getId()));
+        Futures.addCallback(resourceFuture, getCallback(callback, resource -> {
+            if (resource == null) {
+                return ValidationResult.entityNotFound("Resource with requested id wasn't found!");
+            } else {
+                try {
+                    accessControlService.checkPermission(currentUser, Resource.TB_RESOURCE, operation, entityId, resource);
+                } catch (ThingsboardException e) {
+                    return ValidationResult.accessDenied(e.getMessage());
+                }
+                return ValidationResult.ok(resource);
+            }
+        }), executor);
+    }
+
     private void validateAsset(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
         if (currentUser.isSystemAdmin()) {
             callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION));

application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java

@@ -33,8 +33,8 @@ import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.DashboardId;
 import org.thingsboard.server.common.data.id.IdBased;
 import org.thingsboard.server.common.data.id.TenantId;
-import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo;
 import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig;
+import org.thingsboard.server.common.data.oauth2.OAuth2Registration;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.page.PageLink;
 import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
@@ -93,9 +93,9 @@ public abstract class AbstractOAuth2ClientMapper {
     
     private final Lock userCreationLock = new ReentrantLock();
 
-    protected SecurityUser getOrCreateSecurityUserFromOAuth2User(OAuth2User oauth2User, OAuth2ClientRegistrationInfo clientRegistration) {
+    protected SecurityUser getOrCreateSecurityUserFromOAuth2User(OAuth2User oauth2User, OAuth2Registration registration) {
 
-        OAuth2MapperConfig config = clientRegistration.getMapperConfig();
+        OAuth2MapperConfig config = registration.getMapperConfig();
 
         UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, oauth2User.getEmail());
 
@@ -139,9 +139,9 @@ public abstract class AbstractOAuth2ClientMapper {
                         }
                     }
 
-                    if (clientRegistration.getAdditionalInfo() != null &&
-                            clientRegistration.getAdditionalInfo().has("providerName")) {
-                        additionalInfo.put("authProviderName", clientRegistration.getAdditionalInfo().get("providerName").asText());
+                    if (registration.getAdditionalInfo() != null &&
+                            registration.getAdditionalInfo().has("providerName")) {
+                        additionalInfo.put("authProviderName", registration.getAdditionalInfo().get("providerName").asText());
                     }
 
                     user.setAdditionalInfo(additionalInfo);

application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AppleOAuth2ClientMapper.java

@@ -0,0 +1,101 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.security.auth.oauth2;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken;
+import org.springframework.stereotype.Service;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+import org.springframework.util.StringUtils;
+import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig;
+import org.thingsboard.server.common.data.oauth2.OAuth2Registration;
+import org.thingsboard.server.dao.oauth2.OAuth2User;
+import org.thingsboard.server.service.security.model.SecurityUser;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.HashMap;
+import java.util.Map;
+
+@Service(value = "appleOAuth2ClientMapper")
+@Slf4j
+public class AppleOAuth2ClientMapper extends AbstractOAuth2ClientMapper implements OAuth2ClientMapper {
+
+    private static final String USER = "user";
+    private static final String NAME = "name";
+    private static final String FIRST_NAME = "firstName";
+    private static final String LAST_NAME = "lastName";
+    private static final String EMAIL = "email";
+
+    @Override
+    public SecurityUser getOrCreateUserByClientPrincipal(HttpServletRequest request, OAuth2AuthenticationToken token, String providerAccessToken, OAuth2Registration registration) {
+        OAuth2MapperConfig config = registration.getMapperConfig();
+        Map<String, Object> attributes = updateAttributesFromRequestParams(request, token.getPrincipal().getAttributes());
+        String email = BasicMapperUtils.getStringAttributeByKey(attributes, config.getBasic().getEmailAttributeKey());
+        OAuth2User oauth2User = BasicMapperUtils.getOAuth2User(email, attributes, config);
+
+        return getOrCreateSecurityUserFromOAuth2User(oauth2User, registration);
+    }
+
+    private static Map<String, Object> updateAttributesFromRequestParams(HttpServletRequest request, Map<String, Object> attributes) {
+        Map<String, Object> updated = attributes;
+        MultiValueMap<String, String> params = toMultiMap(request.getParameterMap());
+        String userValue = params.getFirst(USER);
+        if (StringUtils.hasText(userValue)) {
+            JsonNode user = null;
+            try {
+                user = JacksonUtil.toJsonNode(userValue);
+            } catch (Exception e) {}
+            if (user != null) {
+                updated = new HashMap<>(attributes);
+                if (user.has(NAME)) {
+                    JsonNode name = user.get(NAME);
+                    if (name.isObject()) {
+                        JsonNode firstName = name.get(FIRST_NAME);
+                        if (firstName != null && firstName.isTextual()) {
+                            updated.put(FIRST_NAME, firstName.asText());
+                        }
+                        JsonNode lastName = name.get(LAST_NAME);
+                        if (lastName != null && lastName.isTextual()) {
+                            updated.put(LAST_NAME, lastName.asText());
+                        }
+                    }
+                }
+                if (user.has(EMAIL)) {
+                    JsonNode email = user.get(EMAIL);
+                    if (email != null && email.isTextual()) {
+                        updated.put(EMAIL, email.asText());
+                    }
+                }
+            }
+        }
+        return updated;
+    }
+
+    private static MultiValueMap<String, String> toMultiMap(Map<String, String[]> map) {
+        MultiValueMap<String, String> params = new LinkedMultiValueMap<>(map.size());
+        map.forEach((key, values) -> {
+            if (values.length > 0) {
+                for (String value : values) {
+                    params.add(key, value);
+                }
+            }
+        });
+        return params;
+    }
+}

application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java

@@ -18,11 +18,12 @@ package org.thingsboard.server.service.security.auth.oauth2;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken;
 import org.springframework.stereotype.Service;
-import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo;
 import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig;
+import org.thingsboard.server.common.data.oauth2.OAuth2Registration;
 import org.thingsboard.server.dao.oauth2.OAuth2User;
 import org.thingsboard.server.service.security.model.SecurityUser;
 
+import javax.servlet.http.HttpServletRequest;
 import java.util.Map;
 
 @Service(value = "basicOAuth2ClientMapper")
@@ -30,12 +31,12 @@ import java.util.Map;
 public class BasicOAuth2ClientMapper extends AbstractOAuth2ClientMapper implements OAuth2ClientMapper {
 
     @Override
-    public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, String providerAccessToken, OAuth2ClientRegistrationInfo clientRegistration) {
-        OAuth2MapperConfig config = clientRegistration.getMapperConfig();
+    public SecurityUser getOrCreateUserByClientPrincipal(HttpServletRequest request, OAuth2AuthenticationToken token, String providerAccessToken, OAuth2Registration registration) {
+        OAuth2MapperConfig config = registration.getMapperConfig();
         Map<String, Object> attributes = token.getPrincipal().getAttributes();
         String email = BasicMapperUtils.getStringAttributeByKey(attributes, config.getBasic().getEmailAttributeKey());
         OAuth2User oauth2User = BasicMapperUtils.getOAuth2User(email, attributes, config);
 
-        return getOrCreateSecurityUserFromOAuth2User(oauth2User, clientRegistration);
+        return getOrCreateSecurityUserFromOAuth2User(oauth2User, registration);
     }
 }

application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java

@@ -23,12 +23,14 @@ import org.springframework.security.oauth2.client.authentication.OAuth2Authentic
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 import org.springframework.web.client.RestTemplate;
-import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo;
 import org.thingsboard.server.common.data.oauth2.OAuth2CustomMapperConfig;
 import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig;
+import org.thingsboard.server.common.data.oauth2.OAuth2Registration;
 import org.thingsboard.server.dao.oauth2.OAuth2User;
 import org.thingsboard.server.service.security.model.SecurityUser;
 
+import javax.servlet.http.HttpServletRequest;
+
 @Service(value = "customOAuth2ClientMapper")
 @Slf4j
 public class CustomOAuth2ClientMapper extends AbstractOAuth2ClientMapper implements OAuth2ClientMapper {
@@ -39,10 +41,10 @@ public class CustomOAuth2ClientMapper extends AbstractOAuth2ClientMapper impleme
     private RestTemplateBuilder restTemplateBuilder = new RestTemplateBuilder();
 
     @Override
-    public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, String providerAccessToken, OAuth2ClientRegistrationInfo clientRegistration) {
-        OAuth2MapperConfig config = clientRegistration.getMapperConfig();
+    public SecurityUser getOrCreateUserByClientPrincipal(HttpServletRequest request, OAuth2AuthenticationToken token, String providerAccessToken, OAuth2Registration registration) {
+        OAuth2MapperConfig config = registration.getMapperConfig();
         OAuth2User oauth2User = getOAuth2User(token, providerAccessToken, config.getCustom());
-        return getOrCreateSecurityUserFromOAuth2User(oauth2User, clientRegistration);
+        return getOrCreateSecurityUserFromOAuth2User(oauth2User, registration);
     }
 
     private synchronized OAuth2User getOAuth2User(OAuth2AuthenticationToken token, String providerAccessToken, OAuth2CustomMapperConfig custom) {

application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/GithubOAuth2ClientMapper.java

@@ -23,12 +23,13 @@ import org.springframework.boot.web.client.RestTemplateBuilder;
 import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken;
 import org.springframework.stereotype.Service;
 import org.springframework.web.client.RestTemplate;
-import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo;
 import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig;
+import org.thingsboard.server.common.data.oauth2.OAuth2Registration;
 import org.thingsboard.server.dao.oauth2.OAuth2Configuration;
 import org.thingsboard.server.dao.oauth2.OAuth2User;
 import org.thingsboard.server.service.security.model.SecurityUser;
 
+import javax.servlet.http.HttpServletRequest;
 import java.util.ArrayList;
 import java.util.Map;
 import java.util.Optional;
@@ -46,13 +47,13 @@ public class GithubOAuth2ClientMapper extends AbstractOAuth2ClientMapper impleme
     private OAuth2Configuration oAuth2Configuration;
 
     @Override
-    public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, String providerAccessToken, OAuth2ClientRegistrationInfo clientRegistration) {
-        OAuth2MapperConfig config = clientRegistration.getMapperConfig();
+    public SecurityUser getOrCreateUserByClientPrincipal(HttpServletRequest request, OAuth2AuthenticationToken token, String providerAccessToken, OAuth2Registration registration) {
+        OAuth2MapperConfig config = registration.getMapperConfig();
         Map<String, String> githubMapperConfig = oAuth2Configuration.getGithubMapper();
         String email = getEmail(githubMapperConfig.get(EMAIL_URL_KEY), providerAccessToken);
         Map<String, Object> attributes = token.getPrincipal().getAttributes();
         OAuth2User oAuth2User = BasicMapperUtils.getOAuth2User(email, attributes, config);
-        return getOrCreateSecurityUserFromOAuth2User(oAuth2User, clientRegistration);
+        return getOrCreateSecurityUserFromOAuth2User(oAuth2User, registration);
     }
 
     private synchronized String getEmail(String emailUrl, String oauth2Token) {

application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java

@@ -16,9 +16,12 @@
 package org.thingsboard.server.service.security.auth.oauth2;
 
 import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken;
-import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo;
+import org.thingsboard.server.common.data.oauth2.OAuth2Registration;
+import org.thingsboard.server.common.data.oauth2.deprecated.OAuth2ClientRegistrationInfo;
 import org.thingsboard.server.service.security.model.SecurityUser;
 
+import javax.servlet.http.HttpServletRequest;
+
 public interface OAuth2ClientMapper {
-    SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, String providerAccessToken, OAuth2ClientRegistrationInfo clientRegistration);
+    SecurityUser getOrCreateUserByClientPrincipal(HttpServletRequest request, OAuth2AuthenticationToken token, String providerAccessToken, OAuth2Registration registration);
 }

application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java

@@ -37,6 +37,10 @@ public class OAuth2ClientMapperProvider {
     @Qualifier("githubOAuth2ClientMapper")
     private OAuth2ClientMapper githubOAuth2ClientMapper;
 
+    @Autowired
+    @Qualifier("appleOAuth2ClientMapper")
+    private OAuth2ClientMapper appleOAuth2ClientMapper;
+
     public OAuth2ClientMapper getOAuth2ClientMapperByType(MapperType oauth2MapperType) {
         switch (oauth2MapperType) {
             case CUSTOM:
@@ -45,6 +49,8 @@ public class OAuth2ClientMapperProvider {
                 return basicOAuth2ClientMapper;
             case GITHUB:
                 return githubOAuth2ClientMapper;
+            case APPLE:
+                return appleOAuth2ClientMapper;
             default:
                 throw new RuntimeException("OAuth2ClientRegistrationMapper with type " + oauth2MapperType + " is not supported!");
         }

application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java

@@ -18,8 +18,10 @@ package org.thingsboard.server.service.security.auth.oauth2;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.security.core.AuthenticationException;
+import org.springframework.security.oauth2.core.endpoint.OAuth2AuthorizationRequest;
 import org.springframework.security.web.authentication.SimpleUrlAuthenticationFailureHandler;
 import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.StringUtils;
 import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
@@ -34,7 +36,6 @@ import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 
 @Component(value = "oauth2AuthenticationFailureHandler")
-@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true")
 public class Oauth2AuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler {
 
     private final HttpCookieOAuth2AuthorizationRequestRepository httpCookieOAuth2AuthorizationRequestRepository;
@@ -51,9 +52,19 @@ public class Oauth2AuthenticationFailureHandler extends SimpleUrlAuthenticationF
     public void onAuthenticationFailure(HttpServletRequest request,
                                         HttpServletResponse response, AuthenticationException exception)
             throws IOException, ServletException {
-        String baseUrl = this.systemSecurityService.getBaseUrl(TenantId.SYS_TENANT_ID, new CustomerId(EntityId.NULL_UUID), request);
+        String baseUrl;
+        String errorPrefix;
+        OAuth2AuthorizationRequest authorizationRequest = httpCookieOAuth2AuthorizationRequestRepository.loadAuthorizationRequest(request);
+        String callbackUrlScheme = authorizationRequest.getAttribute(TbOAuth2ParameterNames.CALLBACK_URL_SCHEME);
+        if (!StringUtils.isEmpty(callbackUrlScheme)) {
+            baseUrl = callbackUrlScheme + ":";
+            errorPrefix = "/?error=";
+        } else {
+            baseUrl = this.systemSecurityService.getBaseUrl(TenantId.SYS_TENANT_ID, new CustomerId(EntityId.NULL_UUID), request);
+            errorPrefix = "/login?loginError=";
+        }
         httpCookieOAuth2AuthorizationRequestRepository.removeAuthorizationRequestCookies(request, response);
-        getRedirectStrategy().sendRedirect(request, response, baseUrl + "/login?loginError=" +
+        getRedirectStrategy().sendRedirect(request, response, baseUrl + errorPrefix +
                 URLEncoder.encode(exception.getMessage(), StandardCharsets.UTF_8.toString()));
     }
 }

application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java

@@ -20,12 +20,14 @@ import org.springframework.security.core.Authentication;
 import org.springframework.security.oauth2.client.OAuth2AuthorizedClient;
 import org.springframework.security.oauth2.client.OAuth2AuthorizedClientService;
 import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken;
+import org.springframework.security.oauth2.core.endpoint.OAuth2AuthorizationRequest;
 import org.springframework.security.web.authentication.SimpleUrlAuthenticationSuccessHandler;
 import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.StringUtils;
 import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
-import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationInfo;
+import org.thingsboard.server.common.data.oauth2.OAuth2Registration;
 import org.thingsboard.server.common.data.security.model.JwtToken;
 import org.thingsboard.server.dao.oauth2.OAuth2Service;
 import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository;
@@ -72,17 +74,24 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS
     public void onAuthenticationSuccess(HttpServletRequest request,
                                         HttpServletResponse response,
                                         Authentication authentication) throws IOException {
-        String baseUrl = this.systemSecurityService.getBaseUrl(TenantId.SYS_TENANT_ID, new CustomerId(EntityId.NULL_UUID), request);
+        OAuth2AuthorizationRequest authorizationRequest = httpCookieOAuth2AuthorizationRequestRepository.loadAuthorizationRequest(request);
+        String callbackUrlScheme = authorizationRequest.getAttribute(TbOAuth2ParameterNames.CALLBACK_URL_SCHEME);
+        String baseUrl;
+        if (!StringUtils.isEmpty(callbackUrlScheme)) {
+            baseUrl = callbackUrlScheme + ":";
+        } else {
+            baseUrl = this.systemSecurityService.getBaseUrl(TenantId.SYS_TENANT_ID, new CustomerId(EntityId.NULL_UUID), request);
+        }
         try {
             OAuth2AuthenticationToken token = (OAuth2AuthenticationToken) authentication;
 
-            OAuth2ClientRegistrationInfo clientRegistration = oAuth2Service.findClientRegistrationInfo(UUID.fromString(token.getAuthorizedClientRegistrationId()));
+            OAuth2Registration registration = oAuth2Service.findRegistration(UUID.fromString(token.getAuthorizedClientRegistrationId()));
             OAuth2AuthorizedClient oAuth2AuthorizedClient = oAuth2AuthorizedClientService.loadAuthorizedClient(
                     token.getAuthorizedClientRegistrationId(),
                     token.getPrincipal().getName());
-            OAuth2ClientMapper mapper = oauth2ClientMapperProvider.getOAuth2ClientMapperByType(clientRegistration.getMapperConfig().getType());
-            SecurityUser securityUser = mapper.getOrCreateUserByClientPrincipal(token, oAuth2AuthorizedClient.getAccessToken().getTokenValue(),
-                    clientRegistration);
+            OAuth2ClientMapper mapper = oauth2ClientMapperProvider.getOAuth2ClientMapperByType(registration.getMapperConfig().getType());
+            SecurityUser securityUser = mapper.getOrCreateUserByClientPrincipal(request, token, oAuth2AuthorizedClient.getAccessToken().getTokenValue(),
+                    registration);
 
             JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser);
             JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser);
@@ -91,7 +100,13 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS
             getRedirectStrategy().sendRedirect(request, response, baseUrl + "/?accessToken=" + accessToken.getToken() + "&refreshToken=" + refreshToken.getToken());
         } catch (Exception e) {
             clearAuthenticationAttributes(request, response);
-            getRedirectStrategy().sendRedirect(request, response, baseUrl + "/login?loginError=" +
+            String errorPrefix;
+            if (!StringUtils.isEmpty(callbackUrlScheme)) {
+                errorPrefix = "/?error=";
+            } else {
+                errorPrefix = "/login?loginError=";
+            }
+            getRedirectStrategy().sendRedirect(request, response, baseUrl + errorPrefix +
                     URLEncoder.encode(e.getMessage(), StandardCharsets.UTF_8.toString()));
         }
     }

application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/TbOAuth2ParameterNames.java

@@ -0,0 +1,22 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.security.auth.oauth2;
+
+public interface TbOAuth2ParameterNames {
+
+    String CALLBACK_URL_SCHEME = "callback_url_scheme";
+
+}

application/src/main/java/org/thingsboard/server/service/security/model/token/OAuth2AppTokenFactory.java

@@ -0,0 +1,69 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.security.model.token;
+
+import io.jsonwebtoken.Claims;
+import io.jsonwebtoken.ExpiredJwtException;
+import io.jsonwebtoken.Jws;
+import io.jsonwebtoken.Jwts;
+import io.jsonwebtoken.MalformedJwtException;
+import io.jsonwebtoken.SignatureException;
+import io.jsonwebtoken.UnsupportedJwtException;
+import io.micrometer.core.instrument.util.StringUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+
+@Component
+@Slf4j
+public class OAuth2AppTokenFactory {
+
+    private static final String CALLBACK_URL_SCHEME = "callbackUrlScheme";
+
+    private static final long MAX_EXPIRATION_TIME_DIFF_MS = TimeUnit.MINUTES.toMillis(5);
+
+    public String validateTokenAndGetCallbackUrlScheme(String appPackage, String appToken, String appSecret) {
+        Jws<Claims> jwsClaims;
+        try {
+            jwsClaims = Jwts.parser().setSigningKey(appSecret).parseClaimsJws(appToken);
+        }
+        catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException | SignatureException ex) {
+            throw new IllegalArgumentException("Invalid Application token: ", ex);
+        } catch (ExpiredJwtException expiredEx) {
+            throw new IllegalArgumentException("Application token expired", expiredEx);
+        }
+        Claims claims = jwsClaims.getBody();
+        Date expiration = claims.getExpiration();
+        if (expiration == null) {
+            throw new IllegalArgumentException("Application token must have expiration date");
+        }
+        long timeDiff = expiration.getTime() - System.currentTimeMillis();
+        if (timeDiff > MAX_EXPIRATION_TIME_DIFF_MS) {
+            throw new IllegalArgumentException("Application token expiration time can't be longer than 5 minutes");
+        }
+        if (!claims.getIssuer().equals(appPackage)) {
+            throw new IllegalArgumentException("Application token issuer doesn't match application package");
+        }
+        String callbackUrlScheme = claims.get(CALLBACK_URL_SCHEME, String.class);
+        if (StringUtils.isEmpty(callbackUrlScheme)) {
+            throw new IllegalArgumentException("Application token doesn't have callbackUrlScheme");
+        }
+        return callbackUrlScheme;
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java

@@ -41,6 +41,7 @@ public class CustomerUserPermissions extends AbstractPermissions {
         put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker);
         put(Resource.WIDGET_TYPE, widgetsPermissionChecker);
         put(Resource.EDGE, customerEntityPermissionChecker);
+        put(Resource.RPC, rpcPermissionChecker);
     }
 
     private static final PermissionChecker customerEntityPermissionChecker =
@@ -138,4 +139,22 @@ public class CustomerUserPermissions extends AbstractPermissions {
         }
 
     };
+
+    private static final PermissionChecker rpcPermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ) {
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) {
+            if (!super.hasPermission(user, operation, entityId, entity)) {
+                return false;
+            }
+            if (entity.getTenantId() == null || entity.getTenantId().isNullUid()) {
+                return true;
+            }
+            if (!user.getTenantId().equals(entity.getTenantId())) {
+                return false;
+            }
+            return true;
+        }
+    };
 }

application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java

@@ -38,7 +38,9 @@ public enum Resource {
     DEVICE_PROFILE(EntityType.DEVICE_PROFILE),
     API_USAGE_STATE(EntityType.API_USAGE_STATE),
     TB_RESOURCE(EntityType.TB_RESOURCE),
-    EDGE(EntityType.EDGE);
+    OTA_PACKAGE(EntityType.OTA_PACKAGE),
+    EDGE(EntityType.EDGE),
+    RPC(EntityType.RPC);
 
     private final EntityType entityType;
 

application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java

@@ -42,7 +42,9 @@ public class TenantAdminPermissions extends AbstractPermissions {
         put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker);
         put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker);
         put(Resource.TB_RESOURCE, tbResourcePermissionChecker);
+        put(Resource.OTA_PACKAGE, tenantEntityPermissionChecker);
         put(Resource.EDGE, tenantEntityPermissionChecker);
+        put(Resource.RPC, tenantEntityPermissionChecker);
     }
 
     public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() {

application/src/main/java/org/thingsboard/server/service/sms/DefaultSmsService.java

@@ -22,6 +22,7 @@ import org.springframework.stereotype.Service;
 import org.thingsboard.rule.engine.api.SmsService;
 import org.thingsboard.rule.engine.api.sms.SmsSender;
 import org.thingsboard.rule.engine.api.sms.SmsSenderFactory;
+import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.sms.config.SmsProviderConfiguration;
 import org.thingsboard.server.common.data.sms.config.TestSmsRequest;
 import org.thingsboard.server.common.data.AdminSettings;
@@ -94,7 +95,7 @@ public class DefaultSmsService implements SmsService {
     }
 
     @Override
-    public void sendSms(TenantId tenantId, String[] numbersTo, String message) throws ThingsboardException {
+    public void sendSms(TenantId tenantId, CustomerId customerId, String[] numbersTo, String message) throws ThingsboardException {
         if (apiUsageStateService.getApiUsageState(tenantId).isSmsSendEnabled()) {
             int smsCount = 0;
             try {
@@ -103,7 +104,7 @@ public class DefaultSmsService implements SmsService {
                 }
             } finally {
                 if (smsCount > 0) {
-                    apiUsageClient.report(tenantId, ApiUsageRecordKey.SMS_EXEC_COUNT, smsCount);
+                    apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.SMS_EXEC_COUNT, smsCount);
                 }
             }
         } else {

application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@@ -54,7 +54,7 @@ import org.thingsboard.server.dao.tenant.TenantService;
 import org.thingsboard.server.dao.timeseries.TimeseriesService;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.gen.transport.TransportProtos;
-import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
+import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
 import org.thingsboard.server.queue.discovery.PartitionService;
 import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
 import org.thingsboard.server.queue.util.TbCoreComponent;
@@ -468,6 +468,7 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
                     md.putValue("deviceName", device.getName());
                     md.putValue("deviceType", device.getType());
                     return DeviceStateData.builder()
+                            .customerId(device.getCustomerId())
                             .tenantId(device.getTenantId())
                             .deviceId(device.getId())
                             .deviceCreationTime(device.getCreatedTime())
@@ -507,7 +508,7 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
             if(!persistToTelemetry){
                 md.putValue(DataConstants.SCOPE, SERVER_SCOPE);
             }
-            TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), md, TbMsgDataType.JSON, data);
+            TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getCustomerId(), md, TbMsgDataType.JSON, data);
             clusterService.pushMsgToRuleEngine(stateData.getTenantId(), stateData.getDeviceId(), tbMsg, null);
         } catch (Exception e) {
             log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e);

application/src/main/java/org/thingsboard/server/service/state/DeviceStateData.java

@@ -17,6 +17,7 @@ package org.thingsboard.server.service.state;
 
 import lombok.Builder;
 import lombok.Data;
+import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.msg.TbMsgMetaData;
@@ -29,6 +30,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData;
 class DeviceStateData {
 
     private final TenantId tenantId;
+    private final CustomerId customerId;
     private final DeviceId deviceId;
     private final long deviceCreationTime;
     private TbMsgMetaData metaData;