tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #13786 from dashevchenko/jsonAttrType 

Added JSON ts type support for bulk import
pull/14156/head
Viacheslav Klimov 8 months ago
committed by GitHub
parent
423255e0ca 6ae8cdf623
commit
e39944ee9d
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
4 changed files with 96 additions and 1 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 9
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java
  2. 19
      application/src/main/java/org/thingsboard/server/utils/CsvUtils.java
  3. 57
      application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java
  4. 12
      common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java

9
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java
View File

@ -17,6 +17,7 @@ package org.thingsboard.server.service.sync.ie.importing.csv;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.FutureCallback;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import jakarta.annotation.Nullable;
@ -183,7 +184,13 @@ public abstract class AbstractBulkImportService<E extends HasId<? extends Entity
data.entrySet().stream()
.filter(dataEntry -> dataEntry.getKey().getType() == kvType &&
StringUtils.isNotEmpty(dataEntry.getKey().getKey()))
.forEach(dataEntry -> kvs.add(dataEntry.getKey().getKey(), dataEntry.getValue().toJsonPrimitive()));
.forEach(dataEntry -> {
ParsedValue value = dataEntry.getValue();
JsonElement kvValue = (value.getDataType() == DataType.JSON)
? (JsonElement) value.getValue()
: value.toJsonPrimitive();
kvs.add(dataEntry.getKey().getKey(), kvValue);
});
return Map.entry(kvType, kvs);
})
.filter(kvsEntry -> kvsEntry.getValue().entrySet().size() > 0)

19
application/src/main/java/org/thingsboard/server/utils/CsvUtils.java
View File

@ -17,10 +17,15 @@ package org.thingsboard.server.utils;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.SneakyThrows;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.io.input.CharSequenceReader;
import java.io.ByteArrayOutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -43,4 +48,18 @@ public class CsvUtils {
.collect(Collectors.toList());
}
@SneakyThrows
public static byte[] generateCsv(List<List<String>> rows) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
try (OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8);
CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT)) {
for (List<String> row : rows) {
csvPrinter.printRecord(row);
}
csvPrinter.flush();
}
return out.toByteArray();
}
}

57
application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java
View File

@ -38,6 +38,7 @@ import org.springframework.test.context.ContextConfiguration;
import org.testcontainers.shaded.org.awaitility.Awaitility;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceInfo;
@ -59,6 +60,7 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceCredentialsId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.relation.EntityRelation;
@ -77,10 +79,15 @@ import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.service.gateway_device.GatewayNotificationsService;
import org.thingsboard.server.service.state.DeviceStateService;
import org.thingsboard.server.utils.CsvUtils;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
@ -1586,6 +1593,56 @@ public class DeviceControllerTest extends AbstractControllerTest {
Assert.assertEquals(newAttributeValue, actualAttribute.get("value"));
}
@Test
public void testBulkImportDeviceWithJsonAttr() throws Exception {
String deviceName = "some_device";
String deviceType = "some_type";
String deviceAttr = "{\"threshold\":45}";
List<List<String>> content = new LinkedList<>();
content.add(Arrays.asList("NAME", "TYPE", "ATTR"));
content.add(Arrays.asList(deviceName, deviceType, deviceAttr));
byte[] bytes = CsvUtils.generateCsv(content);
BulkImportRequest request = new BulkImportRequest();
request.setFile(new String(bytes, StandardCharsets.UTF_8));
BulkImportRequest.Mapping mapping = new BulkImportRequest.Mapping();
BulkImportRequest.ColumnMapping name = new BulkImportRequest.ColumnMapping();
name.setType(BulkImportColumnType.NAME);
BulkImportRequest.ColumnMapping type = new BulkImportRequest.ColumnMapping();
type.setType(BulkImportColumnType.TYPE);
BulkImportRequest.ColumnMapping attr = new BulkImportRequest.ColumnMapping();
attr.setType(BulkImportColumnType.SERVER_ATTRIBUTE);
attr.setKey("attr");
List<BulkImportRequest.ColumnMapping> columns = new ArrayList<>();
columns.add(name);
columns.add(type);
columns.add(attr);
mapping.setColumns(columns);
mapping.setDelimiter(',');
mapping.setUpdate(true);
mapping.setHeader(true);
request.setMapping(mapping);
BulkImportResult<Device> deviceBulkImportResult = doPostWithTypedResponse("/api/device/bulk_import", request, new TypeReference<>() {});
Assert.assertEquals(1, deviceBulkImportResult.getCreated().get());
Assert.assertEquals(0, deviceBulkImportResult.getErrors().get());
Assert.assertEquals(0, deviceBulkImportResult.getUpdated().get());
Assert.assertTrue(deviceBulkImportResult.getErrorsList().isEmpty());
Device savedDevice = doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class);
Assert.assertNotNull(savedDevice);
Assert.assertEquals(deviceName, savedDevice.getName());
Assert.assertEquals(deviceType, savedDevice.getType());
Optional<AttributeKvEntry> retrieved = attributesService.find(tenantId, savedDevice.getId(), AttributeScope.SERVER_SCOPE, "attr").get();
assertThat(retrieved.get().getJsonValue().get()).isEqualTo(deviceAttr);
assertThat(retrieved.get().getStrValue()).isNotPresent();
}
@Test
public void testSaveDeviceWithOutdatedVersion() throws Exception {
Device device = createDevice("Device v1.0");

12
common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java
View File

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.common.data.util;
import com.google.gson.JsonParser;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.thingsboard.server.common.data.kv.DataType;
@ -40,6 +41,11 @@ public class TypeCastUtil {
} catch (RuntimeException ignored) {}
} else if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) {
return Pair.of(DataType.BOOLEAN, Boolean.parseBoolean(value));
} else if (looksLikeJson(value)) {
try {
return Pair.of(DataType.JSON, JsonParser.parseString(value));
} catch (Exception ignored) {
}
}
return Pair.of(DataType.STRING, value);
}
@ -70,4 +76,10 @@ public class TypeCastUtil {
return valueAsString.contains(".") && !valueAsString.contains("E") && !valueAsString.contains("e");
}
private static boolean looksLikeJson(String value) {
String trimmed = value.trim();
return (trimmed.startsWith("{") && trimmed.endsWith("}")) ||
(trimmed.startsWith("[") && trimmed.endsWith("]"));
}
}

Write Preview
Loading…
Cancel
Save