Refactor device bulk import

application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java

@@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.TextNode;
 import lombok.SneakyThrows;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 import org.thingsboard.common.util.JacksonUtil;
@@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.DeviceProfileType;
 import org.thingsboard.server.common.data.DeviceTransportType;
 import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials;
 import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MClientCredentials;
+import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode;
 import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration;
 import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
 import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration;
@@ -58,9 +60,9 @@ import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
 import java.util.Collection;
 import java.util.EnumSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Stream;
 
 @Service
 @TbCoreComponent
@@ -96,29 +98,21 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
             device = existingDevice;
         }
 
-        DeviceCredentials deviceCredentials = createDeviceCredentials(fields);
+        DeviceCredentials deviceCredentials;
-        if (deviceCredentials.getCredentialsType() != null) {
+        try {
-            if (deviceCredentials.getCredentialsType() == DeviceCredentialsType.LWM2M_CREDENTIALS) {
+            deviceCredentials = createDeviceCredentials(fields);
-                setUpLwM2mDeviceProfile(user.getTenantId(), device);
+            deviceCredentialsService.formatCredentials(deviceCredentials);
-            }
+        } catch (Exception e) {
-            try {
+            throw new DeviceCredentialsValidationException("Invalid device credentials: " + e.getMessage());
-                device = deviceService.saveDeviceWithCredentials(device, deviceCredentials);
-            } catch (DeviceCredentialsValidationException e) {
-                if (deviceCredentials.getId() == null) {
-                    device.setId(deviceCredentials.getDeviceId());
-                    importedEntityInfo.setRelatedError("Failed to create " + deviceCredentials.getCredentialsType() + " credentials: "
-                            + e.getMessage() + ". Falling back to access token creds");
-                    deviceService.createAccessTokenCredentials(device, null);
-                } else {
-                    importedEntityInfo.setRelatedError("Failed to update credentials: " + e.getMessage());
-                }
-            }
-        } else {
-            device = deviceService.saveDevice(device);
         }
 
-        importedEntityInfo.setEntity(device);
+        if (deviceCredentials.getCredentialsType() == DeviceCredentialsType.LWM2M_CREDENTIALS) {
+            setUpLwM2mDeviceProfile(user.getTenantId(), device);
+        }
+
+        device = deviceService.saveDeviceWithCredentials(device, deviceCredentials);
 
+        importedEntityInfo.setEntity(device);
         return importedEntityInfo;
     }
 
@@ -148,54 +142,79 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
 
     @SneakyThrows
     private DeviceCredentials createDeviceCredentials(Map<BulkImportColumnType, String> fields) {
-        Set<BulkImportColumnType> columns = fields.keySet();
-
         DeviceCredentials credentials = new DeviceCredentials();
-
+        if (fields.containsKey(BulkImportColumnType.LWM2M_CLIENT_ENDPOINT)) {
-        if (columns.contains(BulkImportColumnType.LWM2M_CLIENT_ENDPOINT)) {
             credentials.setCredentialsType(DeviceCredentialsType.LWM2M_CREDENTIALS);
-            ObjectNode lwm2mCredentials = JacksonUtil.newObjectNode();
+            setUpLwm2mCredentials(fields, credentials);
-
+        } else if (fields.containsKey(BulkImportColumnType.X509)) {
-            ObjectNode client = JacksonUtil.newObjectNode();
-            setValues(client, fields, Set.of(BulkImportColumnType.LWM2M_CLIENT_SECURITY_CONFIG_MODE,
-                    BulkImportColumnType.LWM2M_CLIENT_ENDPOINT, BulkImportColumnType.LWM2M_CLIENT_IDENTITY,
-                    BulkImportColumnType.LWM2M_CLIENT_KEY, BulkImportColumnType.LWM2M_CLIENT_CERT));
-            LwM2MClientCredentials lwM2MClientCredentials = JacksonUtil.treeToValue(client, LwM2MClientCredentials.class);
-            // so that only fields needed for specific type of lwM2MClientCredentials were saved in json
-            lwm2mCredentials.set("client", JacksonUtil.valueToTree(lwM2MClientCredentials));
-
-            ObjectNode bootstrapServer = JacksonUtil.newObjectNode();
-            setValues(bootstrapServer, fields, Set.of(BulkImportColumnType.LWM2M_BOOTSTRAP_SERVER_SECURITY_MODE,
-                    BulkImportColumnType.LWM2M_BOOTSTRAP_SERVER_PUBLIC_KEY_OR_ID, BulkImportColumnType.LWM2M_BOOTSTRAP_SERVER_SECRET_KEY));
-
-            ObjectNode lwm2mServer = JacksonUtil.newObjectNode();
-            setValues(lwm2mServer, fields, Set.of(BulkImportColumnType.LWM2M_SERVER_SECURITY_MODE,
-                    BulkImportColumnType.LWM2M_SERVER_CLIENT_PUBLIC_KEY_OR_ID, BulkImportColumnType.LWM2M_SERVER_CLIENT_SECRET_KEY));
-
-            ObjectNode bootstrap = JacksonUtil.newObjectNode();
-            bootstrap.set("bootstrapServer", bootstrapServer);
-            bootstrap.set("lwm2mServer", lwm2mServer);
-            lwm2mCredentials.set("bootstrap", bootstrap);
-
-            credentials.setCredentialsValue(lwm2mCredentials.toString());
-        } else if (columns.contains(BulkImportColumnType.X509)) {
             credentials.setCredentialsType(DeviceCredentialsType.X509_CERTIFICATE);
-            credentials.setCredentialsValue(fields.get(BulkImportColumnType.X509));
+            setUpX509CertificateCredentials(fields, credentials);
-        } else if (CollectionUtils.containsAny(columns, EnumSet.of(BulkImportColumnType.MQTT_CLIENT_ID, BulkImportColumnType.MQTT_USER_NAME, BulkImportColumnType.MQTT_PASSWORD))) {
+        } else if (CollectionUtils.containsAny(fields.keySet(), EnumSet.of(BulkImportColumnType.MQTT_CLIENT_ID, BulkImportColumnType.MQTT_USER_NAME, BulkImportColumnType.MQTT_PASSWORD))) {
             credentials.setCredentialsType(DeviceCredentialsType.MQTT_BASIC);
-
+            setUpBasicMqttCredentials(fields, credentials);
-            BasicMqttCredentials basicMqttCredentials = new BasicMqttCredentials();
+        } else {
-            basicMqttCredentials.setClientId(fields.get(BulkImportColumnType.MQTT_CLIENT_ID));
-            basicMqttCredentials.setUserName(fields.get(BulkImportColumnType.MQTT_USER_NAME));
-            basicMqttCredentials.setPassword(fields.get(BulkImportColumnType.MQTT_PASSWORD));
-            credentials.setCredentialsValue(JacksonUtil.toString(basicMqttCredentials));
-        } else if (columns.contains(BulkImportColumnType.ACCESS_TOKEN)) {
             credentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN);
-            credentials.setCredentialsId(fields.get(BulkImportColumnType.ACCESS_TOKEN));
+            setUpAccessTokenCredentials(fields, credentials);
         }
         return credentials;
     }
 
+    private void setUpAccessTokenCredentials(Map<BulkImportColumnType, String> fields, DeviceCredentials credentials) {
+        credentials.setCredentialsValue(Optional.ofNullable(fields.get(BulkImportColumnType.ACCESS_TOKEN))
+                .orElseGet(() -> RandomStringUtils.randomAlphanumeric(20)));
+    }
+
+    private void setUpBasicMqttCredentials(Map<BulkImportColumnType, String> fields, DeviceCredentials credentials) {
+        BasicMqttCredentials basicMqttCredentials = new BasicMqttCredentials();
+        basicMqttCredentials.setClientId(fields.get(BulkImportColumnType.MQTT_CLIENT_ID));
+        basicMqttCredentials.setUserName(fields.get(BulkImportColumnType.MQTT_USER_NAME));
+        basicMqttCredentials.setPassword(fields.get(BulkImportColumnType.MQTT_PASSWORD));
+        credentials.setCredentialsValue(JacksonUtil.toString(basicMqttCredentials));
+    }
+
+    private void setUpX509CertificateCredentials(Map<BulkImportColumnType, String> fields, DeviceCredentials credentials) {
+        credentials.setCredentialsValue(fields.get(BulkImportColumnType.X509));
+    }
+
+    private void setUpLwm2mCredentials(Map<BulkImportColumnType, String> fields, DeviceCredentials credentials) throws com.fasterxml.jackson.core.JsonProcessingException {
+        ObjectNode lwm2mCredentials = JacksonUtil.newObjectNode();
+
+        Set.of(BulkImportColumnType.LWM2M_CLIENT_SECURITY_CONFIG_MODE, BulkImportColumnType.LWM2M_BOOTSTRAP_SERVER_SECURITY_MODE,
+                BulkImportColumnType.LWM2M_SERVER_SECURITY_MODE).stream()
+                .map(fields::get)
+                .filter(Objects::nonNull)
+                .forEach(securityMode -> {
+                    try {
+                        LwM2MSecurityMode.valueOf(securityMode);
+                    } catch (IllegalArgumentException e) {
+                        throw new DeviceCredentialsValidationException("Unknown LwM2M security mode: " + securityMode);
+                    }
+                });
+
+        ObjectNode client = JacksonUtil.newObjectNode();
+        setValues(client, fields, Set.of(BulkImportColumnType.LWM2M_CLIENT_SECURITY_CONFIG_MODE,
+                BulkImportColumnType.LWM2M_CLIENT_ENDPOINT, BulkImportColumnType.LWM2M_CLIENT_IDENTITY,
+                BulkImportColumnType.LWM2M_CLIENT_KEY, BulkImportColumnType.LWM2M_CLIENT_CERT));
+        LwM2MClientCredentials lwM2MClientCredentials = JacksonUtil.treeToValue(client, LwM2MClientCredentials.class);
+        // so that only fields needed for specific type of lwM2MClientCredentials were saved in json
+        lwm2mCredentials.set("client", JacksonUtil.valueToTree(lwM2MClientCredentials));
+
+        ObjectNode bootstrapServer = JacksonUtil.newObjectNode();
+        setValues(bootstrapServer, fields, Set.of(BulkImportColumnType.LWM2M_BOOTSTRAP_SERVER_SECURITY_MODE,
+                BulkImportColumnType.LWM2M_BOOTSTRAP_SERVER_PUBLIC_KEY_OR_ID, BulkImportColumnType.LWM2M_BOOTSTRAP_SERVER_SECRET_KEY));
+
+        ObjectNode lwm2mServer = JacksonUtil.newObjectNode();
+        setValues(lwm2mServer, fields, Set.of(BulkImportColumnType.LWM2M_SERVER_SECURITY_MODE,
+                BulkImportColumnType.LWM2M_SERVER_CLIENT_PUBLIC_KEY_OR_ID, BulkImportColumnType.LWM2M_SERVER_CLIENT_SECRET_KEY));
+
+        ObjectNode bootstrap = JacksonUtil.newObjectNode();
+        bootstrap.set("bootstrapServer", bootstrapServer);
+        bootstrap.set("lwm2mServer", lwm2mServer);
+        lwm2mCredentials.set("bootstrap", bootstrap);
+
+        credentials.setCredentialsValue(lwm2mCredentials.toString());
+    }
+
     private void setUpLwM2mDeviceProfile(TenantId tenantId, Device device) {
         DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType());
         if (deviceProfile != null) {

common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsService.java

@@ -29,5 +29,7 @@ public interface DeviceCredentialsService {
 
     DeviceCredentials createDeviceCredentials(TenantId tenantId, DeviceCredentials deviceCredentials);
 
+    void formatCredentials(DeviceCredentials deviceCredentials);
+
     void deleteDeviceCredentials(TenantId tenantId, DeviceCredentials deviceCredentials);
 }

common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java

@@ -56,8 +56,6 @@ public interface DeviceService {
 
     Device saveDevice(ProvisionRequest provisionRequest, DeviceProfile profile);
 
-    void createAccessTokenCredentials(Device device, String accessToken);
-
     Device assignDeviceToCustomer(TenantId tenantId, DeviceId deviceId, CustomerId customerId);
 
     Device unassignDeviceFromCustomer(TenantId tenantId, DeviceId deviceId);

common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/LwM2MClientCredentials.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.common.data.device.credentials.lwm2m;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 
@@ -28,6 +29,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
         @JsonSubTypes.Type(value = RPKClientCredentials.class, name = "RPK"),
         @JsonSubTypes.Type(value = X509ClientCredentials.class, name = "X509")
 })
+@JsonIgnoreProperties(ignoreUnknown = true)
 public interface LwM2MClientCredentials {
 
     @JsonIgnore

dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java

@@ -81,20 +81,7 @@ public class DeviceCredentialsServiceImpl extends AbstractEntityService implemen
     }
 
     private DeviceCredentials saveOrUpdate(TenantId tenantId, DeviceCredentials deviceCredentials) {
-        if (deviceCredentials.getCredentialsType() == null) {
+        formatCredentials(deviceCredentials);
-            throw new DataValidationException("Device credentials type should be specified");
-        }
-        switch (deviceCredentials.getCredentialsType()) {
-            case X509_CERTIFICATE:
-                formatCertData(deviceCredentials);
-                break;
-            case MQTT_BASIC:
-                formatSimpleMqttCredentials(deviceCredentials);
-                break;
-            case LWM2M_CREDENTIALS:
-                formatSimpleLwm2mCredentials(deviceCredentials);
-                break;
-        }
         log.trace("Executing updateDeviceCredentials [{}]", deviceCredentials);
         credentialsValidator.validate(deviceCredentials, id -> tenantId);
         try {
@@ -110,6 +97,21 @@ public class DeviceCredentialsServiceImpl extends AbstractEntityService implemen
         }
     }
 
+    @Override
+    public void formatCredentials(DeviceCredentials deviceCredentials) {
+        switch (deviceCredentials.getCredentialsType()) {
+            case X509_CERTIFICATE:
+                formatCertData(deviceCredentials);
+                break;
+            case MQTT_BASIC:
+                formatSimpleMqttCredentials(deviceCredentials);
+                break;
+            case LWM2M_CREDENTIALS:
+                formatSimpleLwm2mCredentials(deviceCredentials);
+                break;
+        }
+    }
+
     private void formatSimpleMqttCredentials(DeviceCredentials deviceCredentials) {
         BasicMqttCredentials mqttCredentials;
         try {

dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java

@@ -238,20 +238,15 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
     private Device doSaveDevice(Device device, String accessToken, boolean doValidate) {
         Device savedDevice = this.saveDeviceWithoutCredentials(device, doValidate);
         if (device.getId() == null) {
-            createAccessTokenCredentials(savedDevice, accessToken);
+            DeviceCredentials deviceCredentials = new DeviceCredentials();
+            deviceCredentials.setDeviceId(new DeviceId(savedDevice.getUuidId()));
+            deviceCredentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN);
+            deviceCredentials.setCredentialsId(!StringUtils.isEmpty(accessToken) ? accessToken : RandomStringUtils.randomAlphanumeric(20));
+            deviceCredentialsService.createDeviceCredentials(savedDevice.getTenantId(), deviceCredentials);
         }
         return savedDevice;
     }
 
-    @Override
-    public void createAccessTokenCredentials(Device device, String accessToken) {
-        DeviceCredentials deviceCredentials = new DeviceCredentials();
-        deviceCredentials.setDeviceId(new DeviceId(device.getUuidId()));
-        deviceCredentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN);
-        deviceCredentials.setCredentialsId(!StringUtils.isEmpty(accessToken) ? accessToken : RandomStringUtils.randomAlphanumeric(20));
-        deviceCredentialsService.createDeviceCredentials(device.getTenantId(), deviceCredentials);
-    }
-
     private Device saveDeviceWithoutCredentials(Device device, boolean doValidate) {
         log.trace("Executing saveDevice [{}]", device);
         if (doValidate) {