Merge branch 'develop/3.6' of github.com:thingsboard/thingsboard into develop/3.6

application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json

@@ -48,8 +48,8 @@
         "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode",
         "name": "Save Client Attributes",
         "debugMode": false,
+        "configurationVersion": 1,
         "configuration": {
-          "version": 1,
           "scope": "CLIENT_SCOPE",
           "notifyDevice": "false",
           "sendAttributesUpdatedNotification": "false",

application/src/main/data/json/tenant/device_profile/rule_chain_template.json

@@ -32,8 +32,8 @@
         "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode",
         "name": "Save Client Attributes",
         "debugMode": false,
+        "configurationVersion": 1,
         "configuration": {
-          "version": 1,
           "scope": "CLIENT_SCOPE",
           "notifyDevice": "false",
           "sendAttributesUpdatedNotification": "false",

application/src/main/data/json/tenant/rule_chains/root_rule_chain.json

@@ -31,8 +31,8 @@
         "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode",
         "name": "Save Client Attributes",
         "debugMode": false,
+        "configurationVersion": 1,
         "configuration": {
-          "version": 1,
           "scope": "CLIENT_SCOPE",
           "notifyDevice": "false",
           "sendAttributesUpdatedNotification": "false",

application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.TbResource;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.TenantId;
-import org.thingsboard.server.common.data.id.WidgetTypeId;
 import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationTemplate;
 import org.thingsboard.server.common.data.rule.RuleChain;
 import org.thingsboard.server.common.data.rule.RuleChainMetaData;
@@ -104,15 +103,15 @@ public class InstallScripts {
     @Autowired
     private ResourceService resourceService;
 
-    private Path getTenantRuleChainsDir() {
+    Path getTenantRuleChainsDir() {
         return Paths.get(getDataDir(), JSON_DIR, TENANT_DIR, RULE_CHAINS_DIR);
     }
 
-    private Path getDeviceProfileDefaultRuleChainTemplateFilePath() {
+    Path getDeviceProfileDefaultRuleChainTemplateFilePath() {
         return Paths.get(getDataDir(), JSON_DIR, TENANT_DIR, DEVICE_PROFILE_DIR, "rule_chain_template.json");
     }
 
-    private Path getEdgeRuleChainsDir() {
+    Path getEdgeRuleChainsDir() {
         return Paths.get(getDataDir(), JSON_DIR, EDGE_DIR, RULE_CHAINS_DIR);
     }
 
@@ -148,8 +147,8 @@ public class InstallScripts {
     }
 
     private void loadRuleChainsFromPath(TenantId tenantId, Path ruleChainsPath) throws IOException {
-        try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(ruleChainsPath, path -> path.toString().endsWith(InstallScripts.JSON_EXT))) {
+        findRuleChainsFromPath(ruleChainsPath)
-            dirStream.forEach(
+                .forEach(
                     path -> {
                         try {
                             createRuleChainFromFile(tenantId, path, null);
@@ -157,9 +156,15 @@ public class InstallScripts {
                             log.error("Unable to load rule chain from json: [{}]", path.toString());
                             throw new RuntimeException("Unable to load rule chain from json", e);
                         }
-                    }
+                    });
-            );
+    }
+
+    List<Path> findRuleChainsFromPath(Path ruleChainsPath) throws IOException {
+        List<Path> paths = new ArrayList<>();
+        try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(ruleChainsPath, path -> path.toString().endsWith(InstallScripts.JSON_EXT))) {
+            dirStream.forEach(paths::add);
         }
+        return paths;
     }
 
     public RuleChain createDefaultRuleChain(TenantId tenantId, String ruleChainName) throws IOException {

application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java

@@ -66,6 +66,8 @@ import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.PEM_CERT_FI
 
 @TestPropertySource(properties = {
         "device.connectivity.https.enabled=true",
+        "device.connectivity.http.port=8080",
+        "device.connectivity.https.port=444",
         "device.connectivity.mqtts.enabled=true",
         "device.connectivity.mqtts.pem_cert_file=/tmp/" + PEM_CERT_FILE_NAME,
         "device.connectivity.coaps.enabled=true",
@@ -191,7 +193,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest {
         assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://localhost:8080/api/v1/%s/telemetry " +
                         "--header Content-Type:application/json --data \"{temperature:25}\"",
                 credentials.getCredentialsId()));
-        assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://localhost:443/api/v1/%s/telemetry " +
+        assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://localhost:444/api/v1/%s/telemetry " +
                         "--header Content-Type:application/json --data \"{temperature:25}\"",
                 credentials.getCredentialsId()));
 

application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerWithDefaultPortTest.java

@@ -0,0 +1,144 @@
+/**
+ * Copyright © 2016-2023 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.controller;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.AdditionalAnswers;
+import org.mockito.Mockito;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.TestPropertySource;
+import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.common.util.ThingsBoardExecutors;
+import org.thingsboard.server.common.data.Device;
+import org.thingsboard.server.common.data.DeviceProfile;
+import org.thingsboard.server.common.data.DeviceProfileType;
+import org.thingsboard.server.common.data.DeviceTransportType;
+import org.thingsboard.server.common.data.Tenant;
+import org.thingsboard.server.common.data.User;
+import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials;
+import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration;
+import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration;
+import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
+import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration;
+import org.thingsboard.server.common.data.id.DeviceProfileId;
+import org.thingsboard.server.common.data.security.Authority;
+import org.thingsboard.server.common.data.security.DeviceCredentials;
+import org.thingsboard.server.common.data.security.DeviceCredentialsType;
+import org.thingsboard.server.dao.device.DeviceDao;
+import org.thingsboard.server.dao.service.DaoSqlTest;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.COAP;
+import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.COAPS;
+import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.DOCKER;
+import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.HTTP;
+import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.HTTPS;
+import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTT;
+import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTTS;
+import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.PEM_CERT_FILE_NAME;
+
+@TestPropertySource(properties = {
+        "device.connectivity.https.enabled=true",
+        "device.connectivity.http.port=80",
+        "device.connectivity.mqtt.enabled=false",
+        "device.connectivity.mqtts.enabled=false",
+        "device.connectivity.coap.enabled=false",
+        "device.connectivity.coaps.enabled=false",
+})
+@ContextConfiguration(classes = {DeviceConnectivityControllerWithDefaultPortTest.Config.class})
+@DaoSqlTest
+public class DeviceConnectivityControllerWithDefaultPortTest extends AbstractControllerTest {
+
+    ListeningExecutorService executor;
+
+    private Tenant savedTenant;
+
+    static class Config {
+        @Bean
+        @Primary
+        public DeviceDao deviceDao(DeviceDao deviceDao) {
+            return Mockito.mock(DeviceDao.class, AdditionalAnswers.delegatesTo(deviceDao));
+        }
+    }
+
+    @Before
+    public void beforeTest() throws Exception {
+        executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(8, getClass()));
+
+        loginSysAdmin();
+
+        Tenant tenant = new Tenant();
+        tenant.setTitle("My tenant");
+        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        Assert.assertNotNull(savedTenant);
+
+        User tenantAdmin = new User();
+        tenantAdmin.setAuthority(Authority.TENANT_ADMIN);
+        tenantAdmin.setTenantId(savedTenant.getId());
+        tenantAdmin.setEmail("tenant2@thingsboard.org");
+        tenantAdmin.setFirstName("Joe");
+        tenantAdmin.setLastName("Downs");
+
+        createUserAndLogin(tenantAdmin, "testPassword1");
+    }
+
+    @After
+    public void afterTest() throws Exception {
+        executor.shutdownNow();
+
+        loginSysAdmin();
+
+        doDelete("/api/tenant/" + savedTenant.getId().getId())
+                .andExpect(status().isOk());
+    }
+
+    @Test
+    public void testFetchPublishTelemetryCommandsForDefaultDevice() throws Exception {
+        Device device = new Device();
+        device.setName("My device");
+        device.setType("default");
+        Device savedDevice = doPost("/api/device", device, Device.class);
+        JsonNode commands =
+                doGetTyped("/api/device-connectivity/" + savedDevice.getId().getId(), new TypeReference<>() {
+                });
+
+        DeviceCredentials credentials =
+                doGet("/api/device/" + savedDevice.getId().getId() + "/credentials", DeviceCredentials.class);
+
+        assertThat(commands).hasSize(1);
+        JsonNode httpCommands = commands.get(HTTP);
+        assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://localhost/api/v1/%s/telemetry " +
+                        "--header Content-Type:application/json --data \"{temperature:25}\"",
+                credentials.getCredentialsId()));
+        assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://localhost/api/v1/%s/telemetry " +
+                        "--header Content-Type:application/json --data \"{temperature:25}\"",
+                credentials.getCredentialsId()));
+    }
+}

application/src/test/java/org/thingsboard/server/service/install/InstallScriptsTest.java

@@ -0,0 +1,119 @@
+/**
+ * Copyright © 2016-2023 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.install;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.boot.test.mock.mockito.SpyBean;
+import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.server.common.data.id.RuleChainId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.rule.RuleChain;
+import org.thingsboard.server.common.data.rule.RuleChainMetaData;
+import org.thingsboard.server.dao.dashboard.DashboardService;
+import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
+import org.thingsboard.server.dao.resource.ResourceService;
+import org.thingsboard.server.dao.rule.RuleChainService;
+import org.thingsboard.server.dao.service.validator.RuleChainDataValidator;
+import org.thingsboard.server.dao.tenant.TenantService;
+import org.thingsboard.server.dao.usagerecord.ApiLimitService;
+import org.thingsboard.server.dao.widget.WidgetTypeService;
+import org.thingsboard.server.dao.widget.WidgetsBundleService;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.BDDMockito.willReturn;
+
+@Slf4j
+@SpringBootTest(classes = {InstallScripts.class, RuleChainDataValidator.class})
+class InstallScriptsTest {
+
+    @MockBean
+    RuleChainService ruleChainService;
+    @MockBean
+    DashboardService dashboardService;
+    @MockBean
+    WidgetTypeService widgetTypeService;
+    @MockBean
+    WidgetsBundleService widgetsBundleService;
+    @MockBean
+    OAuth2ConfigTemplateService oAuth2TemplateService;
+    @MockBean
+    ResourceService resourceService;
+    @SpyBean
+    InstallScripts installScripts;
+
+    @MockBean
+    TenantService tenantService;
+    @MockBean
+    ApiLimitService apiLimitService;
+    @SpyBean
+    RuleChainDataValidator ruleChainValidator;
+    TenantId tenantId = TenantId.fromUUID(UUID.fromString("9ef79cdf-37a8-4119-b682-2e7ed4e018da"));
+
+    @BeforeEach
+    void setUp() {
+        willReturn(true).given(tenantService).tenantExists(tenantId);
+        willReturn(true).given(apiLimitService).checkEntitiesLimit(any(), any());
+    }
+
+    @Test
+    void testDefaultRuleChainsTemplates() throws IOException {
+        Path dir = installScripts.getTenantRuleChainsDir();
+        installScripts.findRuleChainsFromPath(dir)
+                .forEach(this::validateRuleChainTemplate);
+    }
+
+    @Test
+    void testDefaultEdgeRuleChainsTemplates() throws IOException {
+        Path dir = installScripts.getEdgeRuleChainsDir();
+        installScripts.findRuleChainsFromPath(dir)
+                .forEach(this::validateRuleChainTemplate);
+    }
+
+    @Test
+    void testDeviceProfileDefaultRuleChainTemplate() {
+        validateRuleChainTemplate(installScripts.getDeviceProfileDefaultRuleChainTemplateFilePath());
+    }
+
+    private void validateRuleChainTemplate(Path templateFilePath) {
+        log.warn("validateRuleChainTemplate {}", templateFilePath);
+        JsonNode ruleChainJson = JacksonUtil.toJsonNode(templateFilePath.toFile());
+
+        RuleChain ruleChain = JacksonUtil.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class);
+        ruleChain.setTenantId(tenantId);
+        ruleChainValidator.validate(ruleChain, RuleChain::getTenantId);
+        ruleChain.setId(new RuleChainId(UUID.randomUUID()));
+
+        RuleChainMetaData ruleChainMetaData = JacksonUtil.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class);
+        ruleChainMetaData.setRuleChainId(ruleChain.getId());
+        List<Throwable> throwables = RuleChainDataValidator.validateMetaData(ruleChainMetaData);
+
+        assertThat(throwables).as("templateFilePath " + templateFilePath)
+                .containsExactlyInAnyOrderElementsOf(Collections.emptyList());
+    }
+
+}

dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java

@@ -67,6 +67,8 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService
     public static final String INCORRECT_TENANT_ID = "Incorrect tenantId ";
     public static final String INCORRECT_DEVICE_ID = "Incorrect deviceId ";
     public static final String DEFAULT_DEVICE_TELEMETRY_TOPIC = "v1/devices/me/telemetry";
+    public static final String HTTP_DEFAULT_PORT = "80";
+    public static final String HTTPS_DEFAULT_PORT = "443";
 
     private final Map<String, Resource> certs = new ConcurrentHashMap<>();
 
@@ -203,8 +205,9 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService
             return null;
         }
         String hostName = getHost(baseUrl, properties);
-        String port = properties.getPort().isEmpty() ? "" : ":" + properties.getPort();
+        String propertiesPort = properties.getPort();
-
+        String port = (propertiesPort.isEmpty() || HTTP_DEFAULT_PORT.equals(propertiesPort) || HTTPS_DEFAULT_PORT.equals(propertiesPort))
+                ? "" : ":" + propertiesPort;
         return DeviceConnectivityUtil.getHttpPublishCommand(protocol, hostName, port, deviceCredentials);
     }
 

dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java

@@ -31,7 +31,6 @@ import org.thingsboard.server.common.data.rule.RuleChainType;
 import org.thingsboard.server.common.data.rule.RuleNode;
 import org.thingsboard.server.common.data.util.ReflectionUtils;
 import org.thingsboard.server.dao.exception.DataValidationException;
-import org.thingsboard.server.dao.rule.RuleChainDao;
 import org.thingsboard.server.dao.rule.RuleChainService;
 import org.thingsboard.server.dao.service.ConstraintValidator;
 import org.thingsboard.server.dao.service.DataValidator;
@@ -41,15 +40,14 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 @Component
 @Slf4j
 public class RuleChainDataValidator extends DataValidator<RuleChain> {
 
-    @Autowired
-    private RuleChainDao ruleChainDao;
-
     @Autowired
     @Lazy
     private RuleChainService ruleChainService;
@@ -88,15 +86,19 @@ public class RuleChainDataValidator extends DataValidator<RuleChain> {
         }
     }
 
-    public static void validateMetaData(RuleChainMetaData ruleChainMetaData) {
+    public static List<Throwable> validateMetaData(RuleChainMetaData ruleChainMetaData) {
         ConstraintValidator.validateFields(ruleChainMetaData);
-        ruleChainMetaData.getNodes().forEach(RuleChainDataValidator::validateRuleNode);
+        List<Throwable> throwables = ruleChainMetaData.getNodes().stream()
+                .map(RuleChainDataValidator::validateRuleNode)
+                .filter(Objects::nonNull)
+                .collect(Collectors.toList());
         if (CollectionUtils.isNotEmpty(ruleChainMetaData.getConnections())) {
             validateCircles(ruleChainMetaData.getConnections());
         }
+        return throwables;
     }
 
-    public static void validateRuleNode(RuleNode ruleNode) {
+    public static Throwable validateRuleNode(RuleNode ruleNode) {
         String errorPrefix = "'" + ruleNode.getName() + "' node configuration is invalid: ";
         ConstraintValidator.validateFields(ruleNode, errorPrefix);
         Object nodeConfig;
@@ -104,11 +106,12 @@ public class RuleChainDataValidator extends DataValidator<RuleChain> {
             Class<Object> nodeConfigType = ReflectionUtils.getAnnotationProperty(ruleNode.getType(),
                     "org.thingsboard.rule.engine.api.RuleNode", "configClazz");
             nodeConfig = JacksonUtil.treeToValue(ruleNode.getConfiguration(), nodeConfigType);
-        } catch (Exception e) {
+        } catch (Throwable t) {
-            log.warn("Failed to validate node configuration: {}", ExceptionUtils.getRootCauseMessage(e));
+            log.warn("Failed to validate node configuration: {}", ExceptionUtils.getRootCauseMessage(t));
-            return;
+            return t;
         }
         ConstraintValidator.validateFields(nodeConfig, errorPrefix);
+        return null;
     }
 
     private static void validateCircles(List<NodeConnectionInfo> connectionInfos) {

dao/src/test/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidatorTest.java

@@ -23,7 +23,6 @@ import org.springframework.boot.test.mock.mockito.SpyBean;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.rule.RuleChain;
 import org.thingsboard.server.common.data.rule.RuleChainType;
-import org.thingsboard.server.dao.rule.RuleChainDao;
 import org.thingsboard.server.dao.rule.RuleChainService;
 import org.thingsboard.server.dao.tenant.TenantService;
 
@@ -35,8 +34,6 @@ import static org.mockito.Mockito.verify;
 @SpringBootTest(classes = RuleChainDataValidator.class)
 class RuleChainDataValidatorTest {
 
-    @MockBean
-    RuleChainDao ruleChainDao;
     @MockBean
     RuleChainService ruleChainService;
     @MockBean