tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #14198 from thingsboard/lts-4.2 

LTS to RC
pull/14240/head
Viacheslav Klimov 8 months ago
committed by GitHub
parent
ab7f9b00a2 00fcc0d935
commit
bc174369de
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
10 changed files with 170 additions and 46 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 44
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java
  2. 5
      application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java
  3. 19
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java
  4. 11
      application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java
  5. 2
      application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java
  6. 50
      application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java
  7. 43
      application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java
  8. 7
      common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java
  9. 13
      common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java
  10. 22
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java

44
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java
View File

@ -346,21 +346,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
return mapToArguments(argNames, data);
}
private Map<String, ArgumentEntry> mapToArguments(Map<ReferencedEntityKey, String> argNames, List<TsKvProto> data) {
if (argNames.isEmpty()) {
private Map<String, ArgumentEntry> mapToArguments(Map<ReferencedEntityKey, Set<String>> args, List<TsKvProto> data) {
if (args.isEmpty()) {
return Collections.emptyMap();
}
Map<String, ArgumentEntry> arguments = new HashMap<>();
for (TsKvProto item : data) {
ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null);
String argName = argNames.get(key);
if (argName != null) {
arguments.put(argName, new SingleValueArgumentEntry(item));
Set<String> argNames = args.get(key);
if (argNames != null) {
argNames.forEach(argName -> arguments.put(argName, new SingleValueArgumentEntry(item)));
}
key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null);
argName = argNames.get(key);
if (argName != null) {
arguments.put(argName, new SingleValueArgumentEntry(item));
argNames = args.get(key);
if (argNames != null) {
argNames.forEach(argName -> arguments.put(argName, new SingleValueArgumentEntry(item)));
}
}
return arguments;
@ -378,13 +379,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
return mapToArguments(argNames, scope, attrDataList);
}
private Map<String, ArgumentEntry> mapToArguments(Map<ReferencedEntityKey, String> argNames, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) {
private Map<String, ArgumentEntry> mapToArguments(Map<ReferencedEntityKey, Set<String>> args, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) {
Map<String, ArgumentEntry> arguments = new HashMap<>();
for (AttributeValueProto item : attrDataList) {
ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name()));
String argName = argNames.get(key);
if (argName != null) {
arguments.put(argName, new SingleValueArgumentEntry(item));
Set<String> argNames = args.get(key);
if (argNames != null) {
argNames.forEach(argName -> arguments.put(argName, new SingleValueArgumentEntry(item)));
}
}
return arguments;
@ -402,18 +403,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), scope, removedAttrKeys);
}
private Map<String, ArgumentEntry> mapToArgumentsWithDefaultValue(Map<ReferencedEntityKey, String> argNames, Map<String, Argument> configArguments, AttributeScopeProto scope, List<String> removedAttrKeys) {
private Map<String, ArgumentEntry> mapToArgumentsWithDefaultValue(Map<ReferencedEntityKey, Set<String>> args, Map<String, Argument> configArguments, AttributeScopeProto scope, List<String> removedAttrKeys) {
Map<String, ArgumentEntry> arguments = new HashMap<>();
for (String removedKey : removedAttrKeys) {
ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name()));
String argName = argNames.get(key);
if (argName != null) {
Argument argument = configArguments.get(argName);
String defaultValue = (argument != null) ? argument.getDefaultValue() : null;
arguments.put(argName, StringUtils.isNotEmpty(defaultValue)
? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null)
: new SingleValueArgumentEntry());
Set<String> argNames = args.get(key);
if (argNames != null) {
argNames.forEach(argName -> {
Argument argument = configArguments.get(argName);
String defaultValue = (argument != null) ? argument.getDefaultValue() : null;
arguments.put(argName, StringUtils.isNotEmpty(defaultValue)
? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null)
: new SingleValueArgumentEntry());
});
}
}
return arguments;

5
application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java
View File

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.service.ai;
import com.fasterxml.jackson.core.io.JsonStringEncoder;
import com.google.common.util.concurrent.FluentFuture;
import dev.langchain4j.data.message.ChatMessage;
import dev.langchain4j.data.message.Content;
@ -32,8 +33,6 @@ import org.thingsboard.server.common.data.ai.model.chat.Langchain4jChatModelConf
import java.util.List;
import java.util.stream.Collectors;
import static org.thingsboard.server.common.data.StringUtils.escapeControlChars;
@Service
@RequiredArgsConstructor
class AiChatModelServiceImpl implements AiChatModelService {
@ -74,7 +73,7 @@ class AiChatModelServiceImpl implements AiChatModelService {
private Content prepareContent(Content content) {
if (content instanceof TextContent txt) {
return new TextContent(escapeControlChars(txt.text()));
return new TextContent(new String(JsonStringEncoder.getInstance().quoteAsString(txt.text())));
}
return content;
}

19
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java
View File

@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.util.CollectionsUtil;
import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.dao.usagerecord.ApiLimitService;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto;
@ -44,6 +45,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.thingsboard.common.util.ExpressionFunctionsUtil.userDefinedFunctions;
@ -57,8 +59,8 @@ public class CalculatedFieldCtx {
private EntityId entityId;
private CalculatedFieldType cfType;
private final Map<String, Argument> arguments;
private final Map<ReferencedEntityKey, String> mainEntityArguments;
private final Map<EntityId, Map<ReferencedEntityKey, String>> linkedEntityArguments;
private final Map<ReferencedEntityKey, Set<String>> mainEntityArguments;
private final Map<EntityId, Map<ReferencedEntityKey, Set<String>>> linkedEntityArguments;
private final List<String> argNames;
private Output output;
private String expression;
@ -88,9 +90,10 @@ public class CalculatedFieldCtx {
var refId = entry.getValue().getRefEntityId();
var refKey = entry.getValue().getRefEntityKey();
if (refId == null || refId.equals(calculatedField.getEntityId())) {
mainEntityArguments.put(refKey, entry.getKey());
mainEntityArguments.compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey()));
} else {
linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey());
linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>())
.compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey()));
}
}
this.argNames = new ArrayList<>(arguments.keySet());
@ -182,7 +185,7 @@ public class CalculatedFieldCtx {
return map != null && matchesTimeSeries(map, values);
}
private boolean matchesAttributes(Map<ReferencedEntityKey, String> argMap, List<AttributeKvEntry> values, AttributeScope scope) {
private boolean matchesAttributes(Map<ReferencedEntityKey, Set<String>> argMap, List<AttributeKvEntry> values, AttributeScope scope) {
if (argMap.isEmpty() || values.isEmpty()) {
return false;
}
@ -196,7 +199,7 @@ public class CalculatedFieldCtx {
return false;
}
private boolean matchesTimeSeries(Map<ReferencedEntityKey, String> argMap, List<TsKvEntry> values) {
private boolean matchesTimeSeries(Map<ReferencedEntityKey, Set<String>> argMap, List<TsKvEntry> values) {
if (argMap.isEmpty() || values.isEmpty()) {
return false;
}
@ -225,7 +228,7 @@ public class CalculatedFieldCtx {
return matchesTimeSeriesKeys(mainEntityArguments, keys);
}
private boolean matchesAttributesKeys(Map<ReferencedEntityKey, String> argMap, List<String> keys, AttributeScope scope) {
private boolean matchesAttributesKeys(Map<ReferencedEntityKey, Set<String>> argMap, List<String> keys, AttributeScope scope) {
if (argMap.isEmpty() || keys.isEmpty()) {
return false;
}
@ -240,7 +243,7 @@ public class CalculatedFieldCtx {
return false;
}
private boolean matchesTimeSeriesKeys(Map<ReferencedEntityKey, String> argMap, List<String> keys) {
private boolean matchesTimeSeriesKeys(Map<ReferencedEntityKey, Set<String>> argMap, List<String> keys) {
if (argMap.isEmpty() || keys.isEmpty()) {
return false;
}

11
application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java
View File

@ -186,9 +186,14 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService {
try {
Optional<AttributeKvEntry> provisionState = attributesService.find(device.getTenantId(), device.getId(),
AttributeScope.SERVER_SCOPE, DEVICE_PROVISION_STATE).get();
if (provisionState != null && provisionState.isPresent() && !provisionState.get().getValueAsString().equals(PROVISIONED_STATE)) {
notify(device, provisionRequest, TbMsgType.PROVISION_FAILURE, false);
throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name());
if (provisionState != null && provisionState.isPresent()) {
if (provisionState.get().getValueAsString().equals(PROVISIONED_STATE)) {
notify(device, provisionRequest, TbMsgType.PROVISION_FAILURE, false);
throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name());
} else {
log.error("[{}][{}] Unknown provision state: {}!", device.getName(), DEVICE_PROVISION_STATE, provisionState.get().getValueAsString());
throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name());
}
} else {
saveProvisionStateAttribute(device).get();
notify(device, provisionRequest, TbMsgType.PROVISION_SUCCESS, true);

2
application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java
View File

@ -328,7 +328,7 @@ public class DefaultOtaPackageStateService implements OtaPackageStateService {
attributes.add(new BaseAttributeKvEntry(ts, new LongDataEntry(getAttributeKey(otaPackageType, SIZE), otaPackage.getDataSize())));
}
if (otaPackage.getChecksumAlgorithm() != null) {
if (otaPackage.getChecksumAlgorithm() == null) {
attrToRemove.add(getAttributeKey(otaPackageType, CHECKSUM_ALGORITHM));
} else {
attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, CHECKSUM_ALGORITHM), otaPackage.getChecksumAlgorithm().name())));

50
application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java
View File

@ -659,6 +659,56 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
});
}
@Test
public void testCalculatedFieldWhenTheSameTelemetryKeysUsed() throws Exception {
Device testDevice = createDevice("Test device", "1234567890");
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"a\":5}"));
CalculatedField calculatedField = new CalculatedField();
calculatedField.setEntityId(testDevice.getId());
calculatedField.setType(CalculatedFieldType.SIMPLE);
calculatedField.setName("a + b");
calculatedField.setDebugSettings(DebugSettings.all());
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration();
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("a", ArgumentType.TS_LATEST, null);
Argument argumentA = new Argument();
argumentA.setRefEntityKey(refEntityKey);
Argument argumentB = new Argument();
argumentB.setRefEntityKey(refEntityKey);
config.setArguments(Map.of("a", argumentA, "b", argumentB));
config.setExpression("a + b");
Output output = new Output();
output.setName("c");
output.setType(OutputType.TIME_SERIES);
output.setDecimalsByDefault(0);
config.setOutput(output);
calculatedField.setConfiguration(config);
doPost("/api/calculatedField", calculatedField, CalculatedField.class);
await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode c = getLatestTelemetry(testDevice.getId(), "c");
assertThat(c).isNotNull();
assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("10");
});
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"a\":10}"));
await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode c = getLatestTelemetry(testDevice.getId(), "c");
assertThat(c).isNotNull();
assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("20");
});
}
private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception {
return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class);
}

43
application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java
View File

@ -81,6 +81,7 @@ import org.thingsboard.server.service.state.DeviceStateService;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
@ -387,6 +388,48 @@ public class DeviceControllerTest extends AbstractControllerTest {
.andExpect(statusReason(containsString("Device can`t be referencing to device profile from different tenant!")));
}
@Test
public void testSaveDeviceWithFirmware() throws Exception {
loginTenantAdmin();
DeviceProfile profile = createDeviceProfile("Profile to test ota updates");
profile = doPost("/api/deviceProfile", profile, DeviceProfile.class);
SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest();
firmwareInfo.setDeviceProfileId(profile.getId());
firmwareInfo.setType(FIRMWARE);
String title = "title";
firmwareInfo.setTitle(title);
String fwVersion = "1.0";
firmwareInfo.setVersion(fwVersion);
String url = "test.url";
firmwareInfo.setUrl(url);
firmwareInfo.setUsesUrl(true);
OtaPackageInfo savedFw = doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class);
Device device = new Device();
device.setName("My ota device");
device.setDeviceProfileId(profile.getId());
device.setFirmwareId(savedFw.getId());
device = doPost("/api/device", device, Device.class);
//check shared attributes
Device finalDevice = device;
await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> {
List<Map<String, Object>> attributes = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + finalDevice.getId() +
"/values/attributes/SHARED_SCOPE", new TypeReference<List<Map<String, Object>>>() {
});
return findAttrValue("fw_version", attributes).equals(fwVersion) &&
findAttrValue("fw_title", attributes).equals(title) &&
findAttrValue("fw_url", attributes).equals(url);
});
}
private static Object findAttrValue(String key, List<Map<String, Object>> attributes) {
Optional<Map<String, Object>> attr = attributes.stream()
.filter(att -> att.get("key").equals(key)).findFirst();
return attr.isPresent() ? attr.get().get("value") : "";
}
@Test
public void testSaveDeviceWithFirmwareFromDifferentTenant() throws Exception {
loginDifferentTenant();

7
common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java
View File

@ -275,11 +275,4 @@ public class StringUtils {
return result;
}
public static String escapeControlChars(String text) {
return text
.replace("\n", "\\n")
.replace("\r", "\\r")
.replace("\t", "\\t");
}
}

13
common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java
View File

@ -95,4 +95,17 @@ public class CollectionsUtil {
return false;
}
public static <T> Set<T> addToSet(Set<T> existing, T value) {
if (existing == null || existing.isEmpty()) {
return Set.of(value);
}
if (existing.contains(value)) {
return existing;
}
Set<T> newSet = new HashSet<>(existing.size() + 1);
newSet.addAll(existing);
newSet.add(value);
return (Set<T>) Set.of(newSet.toArray());
}
}

22
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java
View File

@ -21,6 +21,7 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.DeviceProfileProvisionType;
@ -28,6 +29,8 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.msa.AbstractCoapClientTest;
import org.thingsboard.server.msa.DisableUIListeners;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype;
@ -52,14 +55,27 @@ public class CoapClientTest extends AbstractCoapClientTest{
DeviceProfile deviceProfile = testRestClient.getDeviceProfileById(device.getDeviceProfileId());
deviceProfile = updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.CHECK_PRE_PROVISIONED_DEVICES);
DeviceCredentials expectedDeviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId());
DeviceCredentials deviceCreds = testRestClient.getDeviceCredentialsByDeviceId(device.getId());
JsonNode provisionResponse = JacksonUtil.fromBytes(createCoapClientAndPublish(device.getName()));
assertThat(provisionResponse.get("credentialsType").asText()).isEqualTo(expectedDeviceCredentials.getCredentialsType().name());
assertThat(provisionResponse.get("credentialsValue").asText()).isEqualTo(expectedDeviceCredentials.getCredentialsId());
assertThat(provisionResponse.get("credentialsType").asText()).isEqualTo(deviceCreds.getCredentialsType().name());
assertThat(provisionResponse.get("credentialsValue").asText()).isEqualTo(deviceCreds.getCredentialsId());
assertThat(provisionResponse.get("status").asText()).isEqualTo("SUCCESS");
JsonNode attributes = testRestClient.getAttributes(device.getId(), AttributeScope.SERVER_SCOPE, "provisionState");
assertThat(attributes.get(0).get("value").asText()).isEqualTo("provisioned");
// provision second time should fail
JsonNode provisionResponse2 = JacksonUtil.fromBytes(createCoapClientAndPublish(device.getName()));
assertThat(provisionResponse2.get("status").asText()).isEqualTo("FAILURE");
// update provision attribute to non-valid value
testRestClient.postTelemetryAttribute(device.getId(), AttributeScope.SERVER_SCOPE.name(), JacksonUtil.valueToTree(Map.of("provisionState", "non-valid")));
JsonNode provisionResponse3 = JacksonUtil.fromBytes(createCoapClientAndPublish(device.getName()));
assertThat(provisionResponse3.get("status").asText()).isEqualTo("FAILURE");
updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED);
}

Write Preview
Loading…
Cancel
Save