diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 35539834c3..a905738a2f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -346,21 +346,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return mapToArguments(argNames, data); } - private Map mapToArguments(Map argNames, List data) { - if (argNames.isEmpty()) { + private Map mapToArguments(Map> args, List data) { + if (args.isEmpty()) { return Collections.emptyMap(); } Map arguments = new HashMap<>(); for (TsKvProto item : data) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - String argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + Set argNames = args.get(key); + if (argNames != null) { + argNames.forEach(argName -> arguments.put(argName, new SingleValueArgumentEntry(item))); } + key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); - argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + argNames = args.get(key); + if (argNames != null) { + argNames.forEach(argName -> arguments.put(argName, new SingleValueArgumentEntry(item))); } } return arguments; @@ -378,13 +379,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return mapToArguments(argNames, scope, attrDataList); } - private Map mapToArguments(Map argNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(Map> args, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + Set argNames = args.get(key); + if (argNames != null) { + argNames.forEach(argName -> arguments.put(argName, new SingleValueArgumentEntry(item))); } } return arguments; @@ -402,18 +403,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), scope, removedAttrKeys); } - private Map mapToArgumentsWithDefaultValue(Map argNames, Map configArguments, AttributeScopeProto scope, List removedAttrKeys) { + private Map mapToArgumentsWithDefaultValue(Map> args, Map configArguments, AttributeScopeProto scope, List removedAttrKeys) { Map arguments = new HashMap<>(); for (String removedKey : removedAttrKeys) { ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = argNames.get(key); - if (argName != null) { - Argument argument = configArguments.get(argName); - String defaultValue = (argument != null) ? argument.getDefaultValue() : null; - arguments.put(argName, StringUtils.isNotEmpty(defaultValue) - ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) - : new SingleValueArgumentEntry()); - + Set argNames = args.get(key); + if (argNames != null) { + argNames.forEach(argName -> { + Argument argument = configArguments.get(argName); + String defaultValue = (argument != null) ? argument.getDefaultValue() : null; + arguments.put(argName, StringUtils.isNotEmpty(defaultValue) + ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) + : new SingleValueArgumentEntry()); + }); } } return arguments; diff --git a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java index c1829cbf84..639e2025fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.ai; +import com.fasterxml.jackson.core.io.JsonStringEncoder; import com.google.common.util.concurrent.FluentFuture; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.Content; @@ -32,8 +33,6 @@ import org.thingsboard.server.common.data.ai.model.chat.Langchain4jChatModelConf import java.util.List; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.StringUtils.escapeControlChars; - @Service @RequiredArgsConstructor class AiChatModelServiceImpl implements AiChatModelService { @@ -74,7 +73,7 @@ class AiChatModelServiceImpl implements AiChatModelService { private Content prepareContent(Content content) { if (content instanceof TextContent txt) { - return new TextContent(escapeControlChars(txt.text())); + return new TextContent(new String(JsonStringEncoder.getInstance().quoteAsString(txt.text()))); } return content; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index c9eaaef19a..9ef5a8c2a9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -44,6 +45,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import static org.thingsboard.common.util.ExpressionFunctionsUtil.userDefinedFunctions; @@ -57,8 +59,8 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; - private final Map mainEntityArguments; - private final Map> linkedEntityArguments; + private final Map> mainEntityArguments; + private final Map>> linkedEntityArguments; private final List argNames; private Output output; private String expression; @@ -88,9 +90,10 @@ public class CalculatedFieldCtx { var refId = entry.getValue().getRefEntityId(); var refKey = entry.getValue().getRefEntityKey(); if (refId == null || refId.equals(calculatedField.getEntityId())) { - mainEntityArguments.put(refKey, entry.getKey()); + mainEntityArguments.compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey())); } else { - linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); + linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()) + .compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey())); } } this.argNames = new ArrayList<>(arguments.keySet()); @@ -182,7 +185,7 @@ public class CalculatedFieldCtx { return map != null && matchesTimeSeries(map, values); } - private boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { + private boolean matchesAttributes(Map> argMap, List values, AttributeScope scope) { if (argMap.isEmpty() || values.isEmpty()) { return false; } @@ -196,7 +199,7 @@ public class CalculatedFieldCtx { return false; } - private boolean matchesTimeSeries(Map argMap, List values) { + private boolean matchesTimeSeries(Map> argMap, List values) { if (argMap.isEmpty() || values.isEmpty()) { return false; } @@ -225,7 +228,7 @@ public class CalculatedFieldCtx { return matchesTimeSeriesKeys(mainEntityArguments, keys); } - private boolean matchesAttributesKeys(Map argMap, List keys, AttributeScope scope) { + private boolean matchesAttributesKeys(Map> argMap, List keys, AttributeScope scope) { if (argMap.isEmpty() || keys.isEmpty()) { return false; } @@ -240,7 +243,7 @@ public class CalculatedFieldCtx { return false; } - private boolean matchesTimeSeriesKeys(Map argMap, List keys) { + private boolean matchesTimeSeriesKeys(Map> argMap, List keys) { if (argMap.isEmpty() || keys.isEmpty()) { return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java index 0778d61ee7..ffd16c1287 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java @@ -186,9 +186,14 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService { try { Optional provisionState = attributesService.find(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, DEVICE_PROVISION_STATE).get(); - if (provisionState != null && provisionState.isPresent() && !provisionState.get().getValueAsString().equals(PROVISIONED_STATE)) { - notify(device, provisionRequest, TbMsgType.PROVISION_FAILURE, false); - throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + if (provisionState != null && provisionState.isPresent()) { + if (provisionState.get().getValueAsString().equals(PROVISIONED_STATE)) { + notify(device, provisionRequest, TbMsgType.PROVISION_FAILURE, false); + throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + } else { + log.error("[{}][{}] Unknown provision state: {}!", device.getName(), DEVICE_PROVISION_STATE, provisionState.get().getValueAsString()); + throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + } } else { saveProvisionStateAttribute(device).get(); notify(device, provisionRequest, TbMsgType.PROVISION_SUCCESS, true); diff --git a/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java b/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java index 4d9e145711..0b4d3bec6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java @@ -328,7 +328,7 @@ public class DefaultOtaPackageStateService implements OtaPackageStateService { attributes.add(new BaseAttributeKvEntry(ts, new LongDataEntry(getAttributeKey(otaPackageType, SIZE), otaPackage.getDataSize()))); } - if (otaPackage.getChecksumAlgorithm() != null) { + if (otaPackage.getChecksumAlgorithm() == null) { attrToRemove.add(getAttributeKey(otaPackageType, CHECKSUM_ALGORITHM)); } else { attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, CHECKSUM_ALGORITHM), otaPackage.getChecksumAlgorithm().name()))); diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index da4b5758cc..b500f95d45 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -659,6 +659,56 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes }); } + @Test + public void testCalculatedFieldWhenTheSameTelemetryKeysUsed() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"a\":5}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("a + b"); + calculatedField.setDebugSettings(DebugSettings.all()); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("a", ArgumentType.TS_LATEST, null); + Argument argumentA = new Argument(); + argumentA.setRefEntityKey(refEntityKey); + Argument argumentB = new Argument(); + argumentB.setRefEntityKey(refEntityKey); + config.setArguments(Map.of("a", argumentA, "b", argumentB)); + config.setExpression("a + b"); + + Output output = new Output(); + output.setName("c"); + output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); + config.setOutput(output); + + calculatedField.setConfiguration(config); + + doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode c = getLatestTelemetry(testDevice.getId(), "c"); + assertThat(c).isNotNull(); + assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("10"); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"a\":10}")); + + await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode c = getLatestTelemetry(testDevice.getId(), "c"); + assertThat(c).isNotNull(); + assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("20"); + }); + } + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); } diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index 36cede9e84..4551e530d2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -81,6 +81,7 @@ import org.thingsboard.server.service.state.DeviceStateService; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -387,6 +388,48 @@ public class DeviceControllerTest extends AbstractControllerTest { .andExpect(statusReason(containsString("Device can`t be referencing to device profile from different tenant!"))); } + @Test + public void testSaveDeviceWithFirmware() throws Exception { + loginTenantAdmin(); + DeviceProfile profile = createDeviceProfile("Profile to test ota updates"); + profile = doPost("/api/deviceProfile", profile, DeviceProfile.class); + + SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest(); + firmwareInfo.setDeviceProfileId(profile.getId()); + firmwareInfo.setType(FIRMWARE); + String title = "title"; + firmwareInfo.setTitle(title); + String fwVersion = "1.0"; + firmwareInfo.setVersion(fwVersion); + String url = "test.url"; + firmwareInfo.setUrl(url); + firmwareInfo.setUsesUrl(true); + OtaPackageInfo savedFw = doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class); + + Device device = new Device(); + device.setName("My ota device"); + device.setDeviceProfileId(profile.getId()); + device.setFirmwareId(savedFw.getId()); + device = doPost("/api/device", device, Device.class); + + //check shared attributes + Device finalDevice = device; + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> { + List> attributes = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + finalDevice.getId() + + "/values/attributes/SHARED_SCOPE", new TypeReference>>() { + }); + return findAttrValue("fw_version", attributes).equals(fwVersion) && + findAttrValue("fw_title", attributes).equals(title) && + findAttrValue("fw_url", attributes).equals(url); + }); + } + + private static Object findAttrValue(String key, List> attributes) { + Optional> attr = attributes.stream() + .filter(att -> att.get("key").equals(key)).findFirst(); + return attr.isPresent() ? attr.get().get("value") : ""; + } + @Test public void testSaveDeviceWithFirmwareFromDifferentTenant() throws Exception { loginDifferentTenant(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index 2b8b631027..cbc881d72f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -275,11 +275,4 @@ public class StringUtils { return result; } - public static String escapeControlChars(String text) { - return text - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t"); - } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index 71c5256203..082be9b71f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -95,4 +95,17 @@ public class CollectionsUtil { return false; } + public static Set addToSet(Set existing, T value) { + if (existing == null || existing.isEmpty()) { + return Set.of(value); + } + if (existing.contains(value)) { + return existing; + } + Set newSet = new HashSet<>(existing.size() + 1); + newSet.addAll(existing); + newSet.add(value); + return (Set) Set.of(newSet.toArray()); + } + } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java index 995b90e529..5697d40db4 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java @@ -21,6 +21,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; @@ -28,6 +29,8 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.msa.AbstractCoapClientTest; import org.thingsboard.server.msa.DisableUIListeners; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype; @@ -52,14 +55,27 @@ public class CoapClientTest extends AbstractCoapClientTest{ DeviceProfile deviceProfile = testRestClient.getDeviceProfileById(device.getDeviceProfileId()); deviceProfile = updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.CHECK_PRE_PROVISIONED_DEVICES); - DeviceCredentials expectedDeviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); + DeviceCredentials deviceCreds = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); JsonNode provisionResponse = JacksonUtil.fromBytes(createCoapClientAndPublish(device.getName())); - assertThat(provisionResponse.get("credentialsType").asText()).isEqualTo(expectedDeviceCredentials.getCredentialsType().name()); - assertThat(provisionResponse.get("credentialsValue").asText()).isEqualTo(expectedDeviceCredentials.getCredentialsId()); + assertThat(provisionResponse.get("credentialsType").asText()).isEqualTo(deviceCreds.getCredentialsType().name()); + assertThat(provisionResponse.get("credentialsValue").asText()).isEqualTo(deviceCreds.getCredentialsId()); assertThat(provisionResponse.get("status").asText()).isEqualTo("SUCCESS"); + JsonNode attributes = testRestClient.getAttributes(device.getId(), AttributeScope.SERVER_SCOPE, "provisionState"); + assertThat(attributes.get(0).get("value").asText()).isEqualTo("provisioned"); + + // provision second time should fail + JsonNode provisionResponse2 = JacksonUtil.fromBytes(createCoapClientAndPublish(device.getName())); + assertThat(provisionResponse2.get("status").asText()).isEqualTo("FAILURE"); + + // update provision attribute to non-valid value + testRestClient.postTelemetryAttribute(device.getId(), AttributeScope.SERVER_SCOPE.name(), JacksonUtil.valueToTree(Map.of("provisionState", "non-valid"))); + + JsonNode provisionResponse3 = JacksonUtil.fromBytes(createCoapClientAndPublish(device.getName())); + assertThat(provisionResponse3.get("status").asText()).isEqualTo("FAILURE"); + updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED); }