From 1ae1dbef3a77767c1f9b9bbe9437b2222488630c Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Thu, 10 Feb 2022 17:25:53 +0200 Subject: [PATCH 01/26] added DynamicProtoUtils --- .../BaseDeviceProfileControllerTest.java | 170 -------- ...AttributesRequestProtoIntegrationTest.java | 5 +- ...CoapServerSideRpcProtoIntegrationTest.java | 9 +- ...actCoapAttributesProtoIntegrationTest.java | 42 +- ...actCoapTimeseriesProtoIntegrationTest.java | 21 +- ...AbstractMqttAttributesIntegrationTest.java | 5 +- ...tractMqttServerSideRpcIntegrationTest.java | 9 +- ...actMqttAttributesProtoIntegrationTest.java | 6 +- ...actMqttTimeseriesProtoIntegrationTest.java | 16 +- common/data/pom.xml | 4 + .../server/common/data/DynamicProtoUtils.java | 301 ++++++++++++++ .../ProtoTransportPayloadConfiguration.java | 169 +------- .../common/data/DynamicProtoUtilsTest.java | 169 ++++++++ common/queue/pom.xml | 4 - .../transport/adaptor/ProtoConverter.java | 7 +- .../dao/device/DeviceProfileServiceImpl.java | 387 ++++++------------ 16 files changed, 673 insertions(+), 651 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java create mode 100644 common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java index 6e0839b354..0b169ae31e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java @@ -16,12 +16,6 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; import org.apache.commons.lang3.RandomStringUtils; import org.junit.After; import org.junit.Assert; @@ -36,10 +30,8 @@ import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; -import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; @@ -47,13 +39,9 @@ import org.thingsboard.server.common.data.security.Authority; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; public abstract class BaseDeviceProfileControllerTest extends AbstractControllerTest { @@ -513,137 +501,6 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController "}", "[Transport Configuration] invalid attributes proto schema provided! OneOf definition groups don't support!"); } - @Test - public void testSaveProtoDeviceProfileWithMessageNestedTypes() throws Exception { - String schema = "syntax = \"proto3\";\n" + - "\n" + - "package testnested;\n" + - "\n" + - "message Outer {\n" + - " message MiddleAA {\n" + - " message Inner {\n" + - " optional int64 ival = 1;\n" + - " optional bool booly = 2;\n" + - " }\n" + - " Inner inner = 1;\n" + - " }\n" + - " message MiddleBB {\n" + - " message Inner {\n" + - " optional int32 ival = 1;\n" + - " optional bool booly = 2;\n" + - " }\n" + - " Inner inner = 1;\n" + - " }\n" + - " MiddleAA middleAA = 1;\n" + - " MiddleBB middleBB = 2;\n" + - "}"; - DynamicSchema dynamicSchema = getDynamicSchema(schema); - assertNotNull(dynamicSchema); - Set messageTypes = dynamicSchema.getMessageTypes(); - assertEquals(5, messageTypes.size()); - assertTrue(messageTypes.contains("testnested.Outer")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleAA")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleAA.Inner")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleBB")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleBB.Inner")); - - DynamicMessage.Builder middleAAInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); - Descriptors.Descriptor middleAAInnerMsgDescriptor = middleAAInnerMsgBuilder.getDescriptorForType(); - DynamicMessage middleAAInnerMsg = middleAAInnerMsgBuilder - .setField(middleAAInnerMsgDescriptor.findFieldByName("ival"), 1L) - .setField(middleAAInnerMsgDescriptor.findFieldByName("booly"), true) - .build(); - - DynamicMessage.Builder middleAAMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA"); - Descriptors.Descriptor middleAAMsgDescriptor = middleAAMsgBuilder.getDescriptorForType(); - DynamicMessage middleAAMsg = middleAAMsgBuilder - .setField(middleAAMsgDescriptor.findFieldByName("inner"), middleAAInnerMsg) - .build(); - - DynamicMessage.Builder middleBBInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); - Descriptors.Descriptor middleBBInnerMsgDescriptor = middleBBInnerMsgBuilder.getDescriptorForType(); - DynamicMessage middleBBInnerMsg = middleBBInnerMsgBuilder - .setField(middleBBInnerMsgDescriptor.findFieldByName("ival"), 0L) - .setField(middleBBInnerMsgDescriptor.findFieldByName("booly"), false) - .build(); - - DynamicMessage.Builder middleBBMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleBB"); - Descriptors.Descriptor middleBBMsgDescriptor = middleBBMsgBuilder.getDescriptorForType(); - DynamicMessage middleBBMsg = middleBBMsgBuilder - .setField(middleBBMsgDescriptor.findFieldByName("inner"), middleBBInnerMsg) - .build(); - - - DynamicMessage.Builder outerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer"); - Descriptors.Descriptor outerMsgBuilderDescriptor = outerMsgBuilder.getDescriptorForType(); - DynamicMessage outerMsg = outerMsgBuilder - .setField(outerMsgBuilderDescriptor.findFieldByName("middleAA"), middleAAMsg) - .setField(outerMsgBuilderDescriptor.findFieldByName("middleBB"), middleBBMsg) - .build(); - - assertEquals("{\n" + - " \"middleAA\": {\n" + - " \"inner\": {\n" + - " \"ival\": \"1\",\n" + - " \"booly\": true\n" + - " }\n" + - " },\n" + - " \"middleBB\": {\n" + - " \"inner\": {\n" + - " \"ival\": 0,\n" + - " \"booly\": false\n" + - " }\n" + - " }\n" + - "}", dynamicMsgToJson(outerMsgBuilderDescriptor, outerMsg.toByteArray())); - } - - @Test - public void testSaveProtoDeviceProfileWithMessageOneOfs() throws Exception { - String schema = "syntax = \"proto3\";\n" + - "\n" + - "package testoneofs;\n" + - "\n" + - "message SubMessage {\n" + - " repeated string name = 1;\n" + - "}\n" + - "\n" + - "message SampleMessage {\n" + - " optional int32 id = 1;\n" + - " oneof testOneOf {\n" + - " string name = 4;\n" + - " SubMessage subMessage = 9;\n" + - " }\n" + - "}"; - DynamicSchema dynamicSchema = getDynamicSchema(schema); - assertNotNull(dynamicSchema); - Set messageTypes = dynamicSchema.getMessageTypes(); - assertEquals(2, messageTypes.size()); - assertTrue(messageTypes.contains("testoneofs.SubMessage")); - assertTrue(messageTypes.contains("testoneofs.SampleMessage")); - - DynamicMessage.Builder sampleMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SampleMessage"); - Descriptors.Descriptor sampleMsgDescriptor = sampleMsgBuilder.getDescriptorForType(); - assertNotNull(sampleMsgDescriptor); - - List fields = sampleMsgDescriptor.getFields(); - assertEquals(3, fields.size()); - DynamicMessage sampleMsg = sampleMsgBuilder - .setField(sampleMsgDescriptor.findFieldByName("name"), "Bob") - .build(); - assertEquals("{\n" + " \"name\": \"Bob\"\n" + "}", dynamicMsgToJson(sampleMsgDescriptor, sampleMsg.toByteArray())); - - DynamicMessage.Builder subMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SubMessage"); - Descriptors.Descriptor subMsgDescriptor = subMsgBuilder.getDescriptorForType(); - DynamicMessage subMsg = subMsgBuilder - .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "Alice") - .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "John") - .build(); - - DynamicMessage sampleMsgWithOneOfSubMessage = sampleMsgBuilder.setField(sampleMsgDescriptor.findFieldByName("subMessage"), subMsg).build(); - assertEquals("{\n" + " \"subMessage\": {\n" + " \"name\": [\"Alice\", \"John\"]\n" + " }\n" + "}", - dynamicMsgToJson(sampleMsgDescriptor, sampleMsgWithOneOfSubMessage.toByteArray())); - } - @Test public void testSaveProtoDeviceProfileWithInvalidTelemetrySchemaTsField() throws Exception { testSaveDeviceProfileWithInvalidProtoSchema("syntax =\"proto3\";\n" + @@ -835,16 +692,6 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController "}", "[Transport Configuration] invalid rpc request proto schema provided! Failed to get field descriptor for field: params!"); } - private DeviceProfile testSaveDeviceProfileWithProtoPayloadType(String schema) throws Exception { - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema, null, null); - MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration); - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); - DeviceProfile foundDeviceProfile = doGet("/api/deviceProfile/"+savedDeviceProfile.getId().getId().toString(), DeviceProfile.class); - Assert.assertEquals(savedDeviceProfile.getName(), foundDeviceProfile.getName()); - return savedDeviceProfile; - } - private void testSaveDeviceProfileWithInvalidProtoSchema(String schema, String errorMsg) throws Exception { ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema, null, null); MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration); @@ -861,21 +708,4 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController .andExpect(statusReason(containsString(errorMsg))); } - private DynamicSchema getDynamicSchema(String schema) throws Exception { - DeviceProfile deviceProfile = testSaveDeviceProfileWithProtoPayloadType(schema); - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration); - MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttDeviceProfileTransportConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement protoFile = protoTransportPayloadConfiguration.getTransportProtoSchema(schema); - return protoTransportPayloadConfiguration.getDynamicSchema(protoFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - } - - private String dynamicMsgToJson(Descriptors.Descriptor descriptor, byte[] payload) throws InvalidProtocolBufferException { - DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, payload); - return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage); - } - } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/AbstractCoapAttributesRequestProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/AbstractCoapAttributesRequestProtoIntegrationTest.java index 0cbd34ebb7..e331d45d22 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/AbstractCoapAttributesRequestProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/AbstractCoapAttributesRequestProtoIntegrationTest.java @@ -28,6 +28,7 @@ import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -93,8 +94,8 @@ public abstract class AbstractCoapAttributesRequestProtoIntegrationTest extends TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema()); + DynamicSchema attributesSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcProtoIntegrationTest.java index aa79732fde..33a5bafa83 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcProtoIntegrationTest.java @@ -33,6 +33,7 @@ import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -91,8 +92,8 @@ public abstract class AbstractCoapServerSideRpcProtoIntegrationTest extends Abst @Override protected void processOnLoadResponse(CoapResponse response, CoapClient client, Integer observe, CountDownLatch latch) { ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); - ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); - DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); + ProtoFileElement rpcRequestProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcRequestProtoSchema()); + DynamicSchema rpcRequestProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcRequestProtoFileElement, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); byte[] requestPayload = response.getPayload(); DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); @@ -101,8 +102,8 @@ public abstract class AbstractCoapServerSideRpcProtoIntegrationTest extends Abst DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload); Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId"); int requestId = (int) dynamicMessage.getField(requestIdDescriptor); - ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); - DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); + ProtoFileElement rpcResponseProtoSchemaFile = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcResponseProtoSchema()); + DynamicSchema rpcResponseProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); DynamicMessage rpcResponseMsg = rpcResponseBuilder diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesProtoIntegrationTest.java index 0f7e63d1ac..4dc2b1a662 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesProtoIntegrationTest.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -47,18 +48,7 @@ public abstract class AbstractCoapAttributesProtoIntegrationTest extends Abstrac @Test public void testPushAttributes() throws Exception { - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - + DynamicSchema attributesSchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); assertNotNull(nestedJsonObjectBuilderDescriptor); @@ -90,18 +80,7 @@ public abstract class AbstractCoapAttributesProtoIntegrationTest extends Abstrac @Test public void testPushAttributesWithExplicitPresenceProtoKeys() throws Exception { - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - + DynamicSchema attributesSchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); assertNotNull(nestedJsonObjectBuilderDescriptor); @@ -127,4 +106,19 @@ public abstract class AbstractCoapAttributesProtoIntegrationTest extends Abstrac processAttributesTest(Arrays.asList("key1", "key5"), postAttributesMsg.toByteArray(), true); } + private DynamicSchema getDynamicSchema() { + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); + CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; + CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); + assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); + DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + String deviceAttributesProtoSchema = protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceAttributesProtoSchema); + return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + } + } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java index 374555ed81..e5e7b83643 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java @@ -24,6 +24,7 @@ import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -58,8 +59,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -127,8 +129,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(schemaStr); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -182,8 +185,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -249,8 +253,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(schemaStr); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java index 9c6b97bd59..9ef41782cd 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java @@ -29,6 +29,7 @@ import org.eclipse.paho.client.mqttv3.MqttException; import org.eclipse.paho.client.mqttv3.MqttMessage; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; @@ -515,8 +516,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema()); + DynamicSchema attributesSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 00198666e2..32bfead7ac 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -34,6 +34,7 @@ import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.Assert; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; @@ -414,8 +415,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractM MqttMessage message = new MqttMessage(); if (requestTopic.startsWith(MqttTopics.BASE_DEVICE_API_TOPIC) || requestTopic.startsWith(MqttTopics.BASE_DEVICE_API_TOPIC_V2)) { ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); - ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); - DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); + ProtoFileElement rpcRequestProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcRequestProtoSchema()); + DynamicSchema rpcRequestProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcRequestProtoFileElement, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); byte[] requestPayload = mqttMessage.getPayload(); DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); @@ -427,8 +428,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractM for (Descriptors.FieldDescriptor fieldDescriptor: fields) { assertTrue(dynamicMessage.hasField(fieldDescriptor)); } - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); - DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); + ProtoFileElement rpcResponseProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcResponseProtoSchema()); + DynamicSchema rpcResponseProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcResponseProtoFileElement, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java index ef3ff3c8a1..adf67a5c21 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java @@ -22,6 +22,7 @@ import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; @@ -135,8 +136,9 @@ public abstract class AbstractMqttAttributesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA); - return protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + String deviceAttributesProtoSchema = protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceAttributesProtoSchema); + return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); } private DynamicMessage getDefaultDynamicMessage() { diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java index ecb2233332..ac6a30e9b2 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java @@ -25,6 +25,7 @@ import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; @@ -92,7 +93,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac " }\n" + "}"; processBeforeTest("Test Post Telemetry device proto payload", "Test Post Telemetry gateway proto payload", TransportPayloadType.PROTOBUF, POST_DATA_TELEMETRY_TOPIC, null, schemaStr, null, null, null, null, null, DeviceProfileProvisionType.DISABLED, false, false); - DynamicSchema telemetrySchema = getDynamicSchema(schemaStr); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -137,7 +138,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac @Test public void testPushTelemetryWithExplicitPresenceProtoKeys() throws Exception { processBeforeTest("Test Post Telemetry device proto payload", "Test Post Telemetry gateway proto payload", TransportPayloadType.PROTOBUF, POST_DATA_TELEMETRY_TOPIC, null); - DynamicSchema telemetrySchema = getDynamicSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -194,7 +195,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac " }\n" + "}"; processBeforeTest("Test Post Telemetry device proto payload", "Test Post Telemetry gateway proto payload", TransportPayloadType.PROTOBUF, POST_DATA_TELEMETRY_TOPIC, null, schemaStr, null, null, null, null, null, DeviceProfileProvisionType.DISABLED, false, false); - DynamicSchema telemetrySchema = getDynamicSchema(schemaStr); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -280,19 +281,20 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac assertNotNull(device); } - private DynamicSchema getDynamicSchema(String deviceTelemetryProtoSchema) { + private DynamicSchema getDynamicSchema() { DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration); MqttDeviceProfileTransportConfiguration mqttTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(deviceTelemetryProtoSchema); - return protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); } private DynamicMessage getDefaultDynamicMessage() { - DynamicSchema telemetrySchema = getDynamicSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/common/data/pom.xml b/common/data/pom.xml index 450a1cd33d..c9fa24de31 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -104,6 +104,10 @@ io.swagger swagger-annotations + + com.google.protobuf + protobuf-java-util + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java new file mode 100644 index 0000000000..8951222ebd --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java @@ -0,0 +1,301 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.github.os72.protobuf.dynamic.EnumDefinition; +import com.github.os72.protobuf.dynamic.MessageDefinition; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import com.squareup.wire.Syntax; +import com.squareup.wire.schema.Field; +import com.squareup.wire.schema.Location; +import com.squareup.wire.schema.internal.parser.EnumConstantElement; +import com.squareup.wire.schema.internal.parser.EnumElement; +import com.squareup.wire.schema.internal.parser.FieldElement; +import com.squareup.wire.schema.internal.parser.MessageElement; +import com.squareup.wire.schema.internal.parser.OneOfElement; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; +import com.squareup.wire.schema.internal.parser.ProtoParser; +import com.squareup.wire.schema.internal.parser.TypeElement; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +public class DynamicProtoUtils { + + public static final Location LOCATION = new Location("", "", -1, -1); + public static final String PROTO_3_SYNTAX = "proto3"; + + public static Descriptors.Descriptor getDescriptor(String protoSchema, String schemaName) { + try { + DynamicMessage.Builder builder = getDynamicMessageBuilder(protoSchema, schemaName); + return builder.getDescriptorForType(); + } catch (Exception e) { + log.warn("Failed to get Message Descriptor due to {}", e.getMessage()); + return null; + } + } + + public static DynamicMessage.Builder getDynamicMessageBuilder(String protoSchema, String schemaName) { + ProtoFileElement protoFileElement = getProtoFileElement(protoSchema); + DynamicSchema dynamicSchema = getDynamicSchema(protoFileElement, schemaName); + String lastMsgName = getMessageTypes(protoFileElement.getTypes()).stream() + .map(MessageElement::getName).reduce((previous, last) -> last).get(); + return dynamicSchema.newMessageBuilder(lastMsgName); + } + + public static DynamicSchema getDynamicSchema(ProtoFileElement protoFileElement, String schemaName) { + DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); + schemaBuilder.setName(schemaName); + schemaBuilder.setSyntax(PROTO_3_SYNTAX); + schemaBuilder.setPackage(StringUtils.isNotEmpty(protoFileElement.getPackageName()) ? + protoFileElement.getPackageName() : schemaName.toLowerCase()); + List types = protoFileElement.getTypes(); + List messageTypes = getMessageTypes(types); + + if (!messageTypes.isEmpty()) { + List enumTypes = getEnumElements(types); + if (!enumTypes.isEmpty()) { + enumTypes.forEach(enumElement -> { + EnumDefinition enumDefinition = getEnumDefinition(enumElement); + schemaBuilder.addEnumDefinition(enumDefinition); + }); + } + List messageDefinitions = getMessageDefinitions(messageTypes); + messageDefinitions.forEach(schemaBuilder::addMessageDefinition); + try { + return schemaBuilder.build(); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException("Failed to create dynamic schema due to: " + e.getMessage()); + } + } else { + throw new RuntimeException("Failed to get Dynamic Schema! Message types is empty for schema:" + schemaName); + } + } + + public static ProtoFileElement getProtoFileElement(String protoSchema) { + return new ProtoParser(LOCATION, protoSchema.toCharArray()).readProtoFile(); + } + + public static String dynamicMsgToJson(Descriptors.Descriptor descriptor, byte[] payload) throws InvalidProtocolBufferException { + DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, payload); + return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage); + } + + public static DynamicMessage jsonToDynamicMessage(DynamicMessage.Builder builder, String payload) throws InvalidProtocolBufferException { + JsonFormat.parser().ignoringUnknownFields().merge(payload, builder); + return builder.build(); + } + + private static List getMessageTypes(List types) { + return types.stream() + .filter(typeElement -> typeElement instanceof MessageElement) + .map(typeElement -> (MessageElement) typeElement) + .collect(Collectors.toList()); + } + + private static List getEnumElements(List types) { + return types.stream() + .filter(typeElement -> typeElement instanceof EnumElement) + .map(typeElement -> (EnumElement) typeElement) + .collect(Collectors.toList()); + } + + private static List getMessageDefinitions(List messageElementsList) { + if (!messageElementsList.isEmpty()) { + List messageDefinitions = new ArrayList<>(); + messageElementsList.forEach(messageElement -> { + MessageDefinition.Builder messageDefinitionBuilder = MessageDefinition.newBuilder(messageElement.getName()); + + List nestedTypes = messageElement.getNestedTypes(); + if (!nestedTypes.isEmpty()) { + List nestedEnumTypes = getEnumElements(nestedTypes); + if (!nestedEnumTypes.isEmpty()) { + nestedEnumTypes.forEach(enumElement -> { + EnumDefinition nestedEnumDefinition = getEnumDefinition(enumElement); + messageDefinitionBuilder.addEnumDefinition(nestedEnumDefinition); + }); + } + List nestedMessageTypes = getMessageTypes(nestedTypes); + List nestedMessageDefinitions = getMessageDefinitions(nestedMessageTypes); + nestedMessageDefinitions.forEach(messageDefinitionBuilder::addMessageDefinition); + } + List messageElementFields = messageElement.getFields(); + List oneOfs = messageElement.getOneOfs(); + if (!oneOfs.isEmpty()) { + for (OneOfElement oneOfelement : oneOfs) { + MessageDefinition.OneofBuilder oneofBuilder = messageDefinitionBuilder.addOneof(oneOfelement.getName()); + addMessageFieldsToTheOneOfDefinition(oneOfelement.getFields(), oneofBuilder); + } + } + if (!messageElementFields.isEmpty()) { + addMessageFieldsToTheMessageDefinition(messageElementFields, messageDefinitionBuilder); + } + messageDefinitions.add(messageDefinitionBuilder.build()); + }); + return messageDefinitions; + } else { + return Collections.emptyList(); + } + } + + private static EnumDefinition getEnumDefinition(EnumElement enumElement) { + List enumElementTypeConstants = enumElement.getConstants(); + EnumDefinition.Builder enumDefinitionBuilder = EnumDefinition.newBuilder(enumElement.getName()); + if (!enumElementTypeConstants.isEmpty()) { + enumElementTypeConstants.forEach(constantElement -> enumDefinitionBuilder.addValue(constantElement.getName(), constantElement.getTag())); + } + return enumDefinitionBuilder.build(); + } + + + private static void addMessageFieldsToTheMessageDefinition(List messageElementFields, MessageDefinition.Builder messageDefinitionBuilder) { + messageElementFields.forEach(fieldElement -> { + String labelStr = null; + if (fieldElement.getLabel() != null) { + labelStr = fieldElement.getLabel().name().toLowerCase(); + } + messageDefinitionBuilder.addField( + labelStr, + fieldElement.getType(), + fieldElement.getName(), + fieldElement.getTag()); + }); + } + + private static void addMessageFieldsToTheOneOfDefinition(List oneOfsElementFields, MessageDefinition.OneofBuilder oneofBuilder) { + oneOfsElementFields.forEach(fieldElement -> oneofBuilder.addField( + fieldElement.getType(), + fieldElement.getName(), + fieldElement.getTag())); + oneofBuilder.msgDefBuilder(); + } + + // validation + + public static void validateProtoSchema(String schema, String schemaName, String exceptionPrefix) throws IllegalArgumentException { + ProtoParser schemaParser = new ProtoParser(LOCATION, schema.toCharArray()); + ProtoFileElement protoFileElement; + try { + protoFileElement = schemaParser.readProtoFile(); + } catch (Exception e) { + throw new IllegalArgumentException(exceptionPrefix + " failed to parse " + schemaName + " due to: " + e.getMessage()); + } + checkProtoFileSyntax(schemaName, protoFileElement); + checkProtoFileCommonSettings(schemaName, protoFileElement.getOptions().isEmpty(), " Schema options don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, protoFileElement.getPublicImports().isEmpty(), " Schema public imports don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, protoFileElement.getImports().isEmpty(), " Schema imports don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, protoFileElement.getExtendDeclarations().isEmpty(), " Schema extend declarations don't support!", exceptionPrefix); + checkTypeElements(schemaName, protoFileElement, exceptionPrefix); + } + + private static void checkProtoFileSyntax(String schemaName, ProtoFileElement protoFileElement) { + if (protoFileElement.getSyntax() == null || !protoFileElement.getSyntax().equals(Syntax.PROTO_3)) { + throw new IllegalArgumentException("[Transport Configuration] invalid schema syntax: " + protoFileElement.getSyntax() + + " for " + schemaName + " provided! Only " + Syntax.PROTO_3 + " allowed!"); + } + } + + private static void checkProtoFileCommonSettings(String schemaName, boolean isEmptySettings, String invalidSettingsMessage, String exceptionPrefix) { + if (!isEmptySettings) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + invalidSettingsMessage); + } + } + + private static void checkTypeElements(String schemaName, ProtoFileElement protoFileElement, String exceptionPrefix) { + List types = protoFileElement.getTypes(); + if (!types.isEmpty()) { + if (types.stream().noneMatch(typeElement -> typeElement instanceof MessageElement)) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " At least one Message definition should exists!"); + } else { + checkEnumElements(schemaName, getEnumElements(types), exceptionPrefix); + checkMessageElements(schemaName, getMessageTypes(types), exceptionPrefix); + } + } else { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Type elements is empty!"); + } + } + + private static void checkFieldElements(String schemaName, List fieldElements, String exceptionPrefix) { + if (!fieldElements.isEmpty()) { + boolean hasRequiredLabel = fieldElements.stream().anyMatch(fieldElement -> { + Field.Label label = fieldElement.getLabel(); + return label != null && label.equals(Field.Label.REQUIRED); + }); + if (hasRequiredLabel) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Required labels are not supported!"); + } + boolean hasDefaultValue = fieldElements.stream().anyMatch(fieldElement -> fieldElement.getDefaultValue() != null); + if (hasDefaultValue) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Default values are not supported!"); + } + } + } + + private static void checkEnumElements(String schemaName, List enumTypes, String exceptionPrefix) { + if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getNestedTypes().isEmpty())) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Nested types in Enum definitions are not supported!"); + } + if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getOptions().isEmpty())) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Enum definitions options are not supported!"); + } + } + + private static void checkMessageElements(String schemaName, List messageElementsList, String exceptionPrefix) { + if (!messageElementsList.isEmpty()) { + messageElementsList.forEach(messageElement -> { + checkProtoFileCommonSettings(schemaName, messageElement.getGroups().isEmpty(), + " Message definition groups don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, messageElement.getOptions().isEmpty(), + " Message definition options don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, messageElement.getExtensions().isEmpty(), + " Message definition extensions don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, messageElement.getReserveds().isEmpty(), + " Message definition reserved elements don't support!", exceptionPrefix); + checkFieldElements(schemaName, messageElement.getFields(), exceptionPrefix); + List oneOfs = messageElement.getOneOfs(); + if (!oneOfs.isEmpty()) { + oneOfs.forEach(oneOfElement -> { + checkProtoFileCommonSettings(schemaName, oneOfElement.getGroups().isEmpty(), + " OneOf definition groups don't support!", exceptionPrefix); + checkFieldElements(schemaName, oneOfElement.getFields(), exceptionPrefix); + }); + } + List nestedTypes = messageElement.getNestedTypes(); + if (!nestedTypes.isEmpty()) { + List nestedEnumTypes = getEnumElements(nestedTypes); + if (!nestedEnumTypes.isEmpty()) { + checkEnumElements(schemaName, nestedEnumTypes, exceptionPrefix); + } + List nestedMessageTypes = getMessageTypes(nestedTypes); + checkMessageElements(schemaName, nestedMessageTypes, exceptionPrefix); + } + }); + } + } + + public static String invalidSchemaProvidedMessage(String schemaName, String exceptionPrefix) { + return exceptionPrefix + " invalid " + schemaName + " provided!"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java index 409570887c..196069ff5f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java @@ -15,29 +15,15 @@ */ package org.thingsboard.server.common.data.device.profile; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.github.os72.protobuf.dynamic.EnumDefinition; -import com.github.os72.protobuf.dynamic.MessageDefinition; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.squareup.wire.schema.Location; -import com.squareup.wire.schema.internal.parser.EnumConstantElement; -import com.squareup.wire.schema.internal.parser.EnumElement; -import com.squareup.wire.schema.internal.parser.FieldElement; -import com.squareup.wire.schema.internal.parser.MessageElement; -import com.squareup.wire.schema.internal.parser.OneOfElement; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; -import com.squareup.wire.schema.internal.parser.ProtoParser; -import com.squareup.wire.schema.internal.parser.TypeElement; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.DynamicProtoUtils; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TransportPayloadType; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - @Slf4j @Data public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeConfiguration { @@ -63,23 +49,23 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC } public Descriptors.Descriptor getTelemetryDynamicMessageDescriptor(String deviceTelemetryProtoSchema) { - return getDescriptor(deviceTelemetryProtoSchema, TELEMETRY_PROTO_SCHEMA); + return DynamicProtoUtils.getDescriptor(deviceTelemetryProtoSchema, TELEMETRY_PROTO_SCHEMA); } public Descriptors.Descriptor getAttributesDynamicMessageDescriptor(String deviceAttributesProtoSchema) { - return getDescriptor(deviceAttributesProtoSchema, ATTRIBUTES_PROTO_SCHEMA); + return DynamicProtoUtils.getDescriptor(deviceAttributesProtoSchema, ATTRIBUTES_PROTO_SCHEMA); } public Descriptors.Descriptor getRpcResponseDynamicMessageDescriptor(String deviceRpcResponseProtoSchema) { - return getDescriptor(deviceRpcResponseProtoSchema, RPC_RESPONSE_PROTO_SCHEMA); + return DynamicProtoUtils.getDescriptor(deviceRpcResponseProtoSchema, RPC_RESPONSE_PROTO_SCHEMA); } public DynamicMessage.Builder getRpcRequestDynamicMessageBuilder(String deviceRpcRequestProtoSchema) { - return getDynamicMessageBuilder(deviceRpcRequestProtoSchema, RPC_REQUEST_PROTO_SCHEMA); + return DynamicProtoUtils.getDynamicMessageBuilder(deviceRpcRequestProtoSchema, RPC_REQUEST_PROTO_SCHEMA); } public String getDeviceRpcResponseProtoSchema() { - if (!isEmptyStr(deviceRpcResponseProtoSchema)) { + if (StringUtils.isNotEmpty(deviceRpcResponseProtoSchema)) { return deviceRpcResponseProtoSchema; } else { return "syntax =\"proto3\";\n" + @@ -92,7 +78,7 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC } public String getDeviceRpcRequestProtoSchema() { - if (!isEmptyStr(deviceRpcRequestProtoSchema)) { + if (StringUtils.isNotEmpty(deviceRpcRequestProtoSchema)) { return deviceRpcRequestProtoSchema; } else { return "syntax =\"proto3\";\n" + @@ -106,143 +92,4 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC } } - private Descriptors.Descriptor getDescriptor(String protoSchema, String schemaName) { - try { - DynamicMessage.Builder builder = getDynamicMessageBuilder(protoSchema, schemaName); - return builder.getDescriptorForType(); - } catch (Exception e) { - log.warn("Failed to get Message Descriptor due to {}", e.getMessage()); - return null; - } - } - - public DynamicMessage.Builder getDynamicMessageBuilder(String protoSchema, String schemaName) { - ProtoFileElement protoFileElement = getTransportProtoSchema(protoSchema); - DynamicSchema dynamicSchema = getDynamicSchema(protoFileElement, schemaName); - String lastMsgName = getMessageTypes(protoFileElement.getTypes()).stream() - .map(MessageElement::getName).reduce((previous, last) -> last).get(); - return dynamicSchema.newMessageBuilder(lastMsgName); - } - - public DynamicSchema getDynamicSchema(ProtoFileElement protoFileElement, String schemaName) { - DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); - schemaBuilder.setName(schemaName); - schemaBuilder.setSyntax(PROTO_3_SYNTAX); - schemaBuilder.setPackage(!isEmptyStr(protoFileElement.getPackageName()) ? - protoFileElement.getPackageName() : schemaName.toLowerCase()); - List types = protoFileElement.getTypes(); - List messageTypes = getMessageTypes(types); - - if (!messageTypes.isEmpty()) { - List enumTypes = getEnumElements(types); - if (!enumTypes.isEmpty()) { - enumTypes.forEach(enumElement -> { - EnumDefinition enumDefinition = getEnumDefinition(enumElement); - schemaBuilder.addEnumDefinition(enumDefinition); - }); - } - List messageDefinitions = getMessageDefinitions(messageTypes); - messageDefinitions.forEach(schemaBuilder::addMessageDefinition); - try { - return schemaBuilder.build(); - } catch (Descriptors.DescriptorValidationException e) { - throw new RuntimeException("Failed to create dynamic schema due to: " + e.getMessage()); - } - } else { - throw new RuntimeException("Failed to get Dynamic Schema! Message types is empty for schema:" + schemaName); - } - } - - public ProtoFileElement getTransportProtoSchema(String protoSchema) { - return new ProtoParser(LOCATION, protoSchema.toCharArray()).readProtoFile(); - } - - private List getMessageTypes(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof MessageElement) - .map(typeElement -> (MessageElement) typeElement) - .collect(Collectors.toList()); - } - - private List getEnumElements(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof EnumElement) - .map(typeElement -> (EnumElement) typeElement) - .collect(Collectors.toList()); - } - - private List getMessageDefinitions(List messageElementsList) { - if (!messageElementsList.isEmpty()) { - List messageDefinitions = new ArrayList<>(); - messageElementsList.forEach(messageElement -> { - MessageDefinition.Builder messageDefinitionBuilder = MessageDefinition.newBuilder(messageElement.getName()); - - List nestedTypes = messageElement.getNestedTypes(); - if (!nestedTypes.isEmpty()) { - List nestedEnumTypes = getEnumElements(nestedTypes); - if (!nestedEnumTypes.isEmpty()) { - nestedEnumTypes.forEach(enumElement -> { - EnumDefinition nestedEnumDefinition = getEnumDefinition(enumElement); - messageDefinitionBuilder.addEnumDefinition(nestedEnumDefinition); - }); - } - List nestedMessageTypes = getMessageTypes(nestedTypes); - List nestedMessageDefinitions = getMessageDefinitions(nestedMessageTypes); - nestedMessageDefinitions.forEach(messageDefinitionBuilder::addMessageDefinition); - } - List messageElementFields = messageElement.getFields(); - List oneOfs = messageElement.getOneOfs(); - if (!oneOfs.isEmpty()) { - for (OneOfElement oneOfelement : oneOfs) { - MessageDefinition.OneofBuilder oneofBuilder = messageDefinitionBuilder.addOneof(oneOfelement.getName()); - addMessageFieldsToTheOneOfDefinition(oneOfelement.getFields(), oneofBuilder); - } - } - if (!messageElementFields.isEmpty()) { - addMessageFieldsToTheMessageDefinition(messageElementFields, messageDefinitionBuilder); - } - messageDefinitions.add(messageDefinitionBuilder.build()); - }); - return messageDefinitions; - } else { - return Collections.emptyList(); - } - } - - private EnumDefinition getEnumDefinition(EnumElement enumElement) { - List enumElementTypeConstants = enumElement.getConstants(); - EnumDefinition.Builder enumDefinitionBuilder = EnumDefinition.newBuilder(enumElement.getName()); - if (!enumElementTypeConstants.isEmpty()) { - enumElementTypeConstants.forEach(constantElement -> enumDefinitionBuilder.addValue(constantElement.getName(), constantElement.getTag())); - } - return enumDefinitionBuilder.build(); - } - - - private void addMessageFieldsToTheMessageDefinition(List messageElementFields, MessageDefinition.Builder messageDefinitionBuilder) { - messageElementFields.forEach(fieldElement -> { - String labelStr = null; - if (fieldElement.getLabel() != null) { - labelStr = fieldElement.getLabel().name().toLowerCase(); - } - messageDefinitionBuilder.addField( - labelStr, - fieldElement.getType(), - fieldElement.getName(), - fieldElement.getTag()); - }); - } - - private void addMessageFieldsToTheOneOfDefinition(List oneOfsElementFields, MessageDefinition.OneofBuilder oneofBuilder) { - oneOfsElementFields.forEach(fieldElement -> oneofBuilder.addField( - fieldElement.getType(), - fieldElement.getName(), - fieldElement.getTag())); - oneofBuilder.msgDefBuilder(); - } - - private boolean isEmptyStr(String str) { - return str == null || "".equals(str); - } - } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java new file mode 100644 index 0000000000..9ae4651cc9 --- /dev/null +++ b/common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java @@ -0,0 +1,169 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class DynamicProtoUtilsTest { + + @Test + public void testProtoSchemaWithMessageNestedTypes() throws Exception { + String schema = "syntax = \"proto3\";\n" + + "\n" + + "package testnested;\n" + + "\n" + + "message Outer {\n" + + " message MiddleAA {\n" + + " message Inner {\n" + + " optional int64 ival = 1;\n" + + " optional bool booly = 2;\n" + + " }\n" + + " Inner inner = 1;\n" + + " }\n" + + " message MiddleBB {\n" + + " message Inner {\n" + + " optional int32 ival = 1;\n" + + " optional bool booly = 2;\n" + + " }\n" + + " Inner inner = 1;\n" + + " }\n" + + " MiddleAA middleAA = 1;\n" + + " MiddleBB middleBB = 2;\n" + + "}"; + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(schema); + DynamicSchema dynamicSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, "test schema with nested types"); + assertNotNull(dynamicSchema); + Set messageTypes = dynamicSchema.getMessageTypes(); + assertEquals(5, messageTypes.size()); + assertTrue(messageTypes.contains("testnested.Outer")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleAA")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleAA.Inner")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleBB")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleBB.Inner")); + + DynamicMessage.Builder middleAAInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); + Descriptors.Descriptor middleAAInnerMsgDescriptor = middleAAInnerMsgBuilder.getDescriptorForType(); + DynamicMessage middleAAInnerMsg = middleAAInnerMsgBuilder + .setField(middleAAInnerMsgDescriptor.findFieldByName("ival"), 1L) + .setField(middleAAInnerMsgDescriptor.findFieldByName("booly"), true) + .build(); + + DynamicMessage.Builder middleAAMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA"); + Descriptors.Descriptor middleAAMsgDescriptor = middleAAMsgBuilder.getDescriptorForType(); + DynamicMessage middleAAMsg = middleAAMsgBuilder + .setField(middleAAMsgDescriptor.findFieldByName("inner"), middleAAInnerMsg) + .build(); + + DynamicMessage.Builder middleBBInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); + Descriptors.Descriptor middleBBInnerMsgDescriptor = middleBBInnerMsgBuilder.getDescriptorForType(); + DynamicMessage middleBBInnerMsg = middleBBInnerMsgBuilder + .setField(middleBBInnerMsgDescriptor.findFieldByName("ival"), 0L) + .setField(middleBBInnerMsgDescriptor.findFieldByName("booly"), false) + .build(); + + DynamicMessage.Builder middleBBMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleBB"); + Descriptors.Descriptor middleBBMsgDescriptor = middleBBMsgBuilder.getDescriptorForType(); + DynamicMessage middleBBMsg = middleBBMsgBuilder + .setField(middleBBMsgDescriptor.findFieldByName("inner"), middleBBInnerMsg) + .build(); + + + DynamicMessage.Builder outerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer"); + Descriptors.Descriptor outerMsgBuilderDescriptor = outerMsgBuilder.getDescriptorForType(); + DynamicMessage outerMsg = outerMsgBuilder + .setField(outerMsgBuilderDescriptor.findFieldByName("middleAA"), middleAAMsg) + .setField(outerMsgBuilderDescriptor.findFieldByName("middleBB"), middleBBMsg) + .build(); + + assertEquals("{\n" + + " \"middleAA\": {\n" + + " \"inner\": {\n" + + " \"ival\": \"1\",\n" + + " \"booly\": true\n" + + " }\n" + + " },\n" + + " \"middleBB\": {\n" + + " \"inner\": {\n" + + " \"ival\": 0,\n" + + " \"booly\": false\n" + + " }\n" + + " }\n" + + "}", DynamicProtoUtils.dynamicMsgToJson(outerMsgBuilderDescriptor, outerMsg.toByteArray())); + } + + @Test + public void testProtoSchemaWithMessageOneOfs() throws Exception { + String schema = "syntax = \"proto3\";\n" + + "\n" + + "package testoneofs;\n" + + "\n" + + "message SubMessage {\n" + + " repeated string name = 1;\n" + + "}\n" + + "\n" + + "message SampleMessage {\n" + + " optional int32 id = 1;\n" + + " oneof testOneOf {\n" + + " string name = 4;\n" + + " SubMessage subMessage = 9;\n" + + " }\n" + + "}"; + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(schema); + DynamicSchema dynamicSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, "test schema with message oneOfs"); + assertNotNull(dynamicSchema); + Set messageTypes = dynamicSchema.getMessageTypes(); + assertEquals(2, messageTypes.size()); + assertTrue(messageTypes.contains("testoneofs.SubMessage")); + assertTrue(messageTypes.contains("testoneofs.SampleMessage")); + + DynamicMessage.Builder sampleMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SampleMessage"); + Descriptors.Descriptor sampleMsgDescriptor = sampleMsgBuilder.getDescriptorForType(); + assertNotNull(sampleMsgDescriptor); + + List fields = sampleMsgDescriptor.getFields(); + assertEquals(3, fields.size()); + DynamicMessage sampleMsg = sampleMsgBuilder + .setField(sampleMsgDescriptor.findFieldByName("name"), "Bob") + .build(); + assertEquals("{\n" + " \"name\": \"Bob\"\n" + "}", DynamicProtoUtils.dynamicMsgToJson(sampleMsgDescriptor, sampleMsg.toByteArray())); + + DynamicMessage.Builder subMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SubMessage"); + Descriptors.Descriptor subMsgDescriptor = subMsgBuilder.getDescriptorForType(); + DynamicMessage subMsg = subMsgBuilder + .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "Alice") + .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "John") + .build(); + + DynamicMessage sampleMsgWithOneOfSubMessage = sampleMsgBuilder.setField(sampleMsgDescriptor.findFieldByName("subMessage"), subMsg).build(); + assertEquals("{\n" + " \"subMessage\": {\n" + " \"name\": [\"Alice\", \"John\"]\n" + " }\n" + "}", + DynamicProtoUtils.dynamicMsgToJson(sampleMsgDescriptor, sampleMsgWithOneOfSubMessage.toByteArray())); + } + +} diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 8240a4070f..c273323cb7 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -116,10 +116,6 @@ com.google.protobuf protobuf-java - - com.google.protobuf - protobuf-java-util - org.apache.curator curator-recipes diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java index 137ceff337..0fb9a767eb 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java @@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; @@ -184,8 +185,7 @@ public class ProtoConverter { try { JsonElement paramsElement = JSON_PARSER.parse(params); rpcRequestJson.add("params", paramsElement); - JsonFormat.parser().ignoringUnknownFields().merge(GSON.toJson(rpcRequestJson), rpcRequestDynamicMessageBuilder); - DynamicMessage dynamicRpcRequest = rpcRequestDynamicMessageBuilder.build(); + DynamicMessage dynamicRpcRequest = DynamicProtoUtils.jsonToDynamicMessage(rpcRequestDynamicMessageBuilder, GSON.toJson(rpcRequestJson)); return dynamicRpcRequest.toByteArray(); } catch (Exception e) { throw new AdaptorException("Failed to convert ToDeviceRpcRequestMsg to Dynamic Rpc request message due to: ", e); @@ -200,8 +200,7 @@ public class ProtoConverter { } public static String dynamicMsgToJson(byte[] bytes, Descriptors.Descriptor descriptor) throws InvalidProtocolBufferException { - DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, bytes); - return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage); + return DynamicProtoUtils.dynamicMsgToJson(descriptor, bytes); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 9a272b4739..1f7849b7b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -17,18 +17,9 @@ package org.thingsboard.server.dao.device; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import com.squareup.wire.Syntax; -import com.squareup.wire.schema.Field; -import com.squareup.wire.schema.Location; -import com.squareup.wire.schema.internal.parser.EnumElement; -import com.squareup.wire.schema.internal.parser.FieldElement; -import com.squareup.wire.schema.internal.parser.MessageElement; -import com.squareup.wire.schema.internal.parser.OneOfElement; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; -import com.squareup.wire.schema.internal.parser.ProtoParser; -import com.squareup.wire.schema.internal.parser.TypeElement; import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.core.util.SecurityUtil; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.StringUtils; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; @@ -91,7 +82,6 @@ import java.util.List; import java.util.Set; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; import static org.thingsboard.server.common.data.CacheConstants.DEVICE_PROFILE_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -104,15 +94,11 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D private static final String INCORRECT_DEVICE_PROFILE_ID = "Incorrect deviceProfileId "; private static final String INCORRECT_DEVICE_PROFILE_NAME = "Incorrect deviceProfileName "; - private static final Location LOCATION = new Location("", "", -1, -1); private static final String ATTRIBUTES_PROTO_SCHEMA = "attributes proto schema"; private static final String TELEMETRY_PROTO_SCHEMA = "telemetry proto schema"; private static final String RPC_REQUEST_PROTO_SCHEMA = "rpc request proto schema"; private static final String RPC_RESPONSE_PROTO_SCHEMA = "rpc response proto schema"; - - private static String invalidSchemaProvidedMessage(String schemaName) { - return "[Transport Configuration] invalid " + schemaName + " provided!"; - } + private static final String EXCEPTION_PREFIX = "[Transport Configuration]"; @Autowired(required = false) private QueueService queueService; @@ -386,8 +372,8 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D throw new DataValidationException("Another default device profile is present in scope of current tenant!"); } } - if (!StringUtils.isEmpty(deviceProfile.getDefaultQueueName()) && queueService != null){ - if(!queueService.getQueuesByServiceType(ServiceType.TB_RULE_ENGINE).contains(deviceProfile.getDefaultQueueName())){ + if (!StringUtils.isEmpty(deviceProfile.getDefaultQueueName()) && queueService != null) { + if (!queueService.getQueuesByServiceType(ServiceType.TB_RULE_ENGINE).contains(deviceProfile.getDefaultQueueName())) { throw new DataValidationException("Device profile is referencing to non-existent queue!"); } } @@ -515,270 +501,153 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D private void validateProtoSchemas(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { try { - validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceAttributesProtoSchema(), ATTRIBUTES_PROTO_SCHEMA); - validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema(), TELEMETRY_PROTO_SCHEMA); - validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcRequestProtoSchema(), RPC_REQUEST_PROTO_SCHEMA); - validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcResponseProtoSchema(), RPC_RESPONSE_PROTO_SCHEMA); + DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceAttributesProtoSchema(), ATTRIBUTES_PROTO_SCHEMA, EXCEPTION_PREFIX); + DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema(), TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX); + DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcRequestProtoSchema(), RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX); + DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcResponseProtoSchema(), RPC_RESPONSE_PROTO_SCHEMA, EXCEPTION_PREFIX); } catch (Exception exception) { throw new DataValidationException(exception.getMessage()); } } - private void validateTransportProtoSchema(String schema, String schemaName) throws IllegalArgumentException { - ProtoParser schemaParser = new ProtoParser(LOCATION, schema.toCharArray()); - ProtoFileElement protoFileElement; - try { - protoFileElement = schemaParser.readProtoFile(); - } catch (Exception e) { - throw new IllegalArgumentException("[Transport Configuration] failed to parse " + schemaName + " due to: " + e.getMessage()); - } - checkProtoFileSyntax(schemaName, protoFileElement); - checkProtoFileCommonSettings(schemaName, protoFileElement.getOptions().isEmpty(), " Schema options don't support!"); - checkProtoFileCommonSettings(schemaName, protoFileElement.getPublicImports().isEmpty(), " Schema public imports don't support!"); - checkProtoFileCommonSettings(schemaName, protoFileElement.getImports().isEmpty(), " Schema imports don't support!"); - checkProtoFileCommonSettings(schemaName, protoFileElement.getExtendDeclarations().isEmpty(), " Schema extend declarations don't support!"); - checkTypeElements(schemaName, protoFileElement); - } - - private void checkProtoFileSyntax(String schemaName, ProtoFileElement protoFileElement) { - if (protoFileElement.getSyntax() == null || !protoFileElement.getSyntax().equals(Syntax.PROTO_3)) { - throw new IllegalArgumentException("[Transport Configuration] invalid schema syntax: " + protoFileElement.getSyntax() + - " for " + schemaName + " provided! Only " + Syntax.PROTO_3 + " allowed!"); + private void validateLwm2mServersConfigOfBootstrapForClient(List lwM2MBootstrapServersConfigurations, boolean isBootstrapServerUpdateEnable) { + Set uris = new HashSet<>(); + Set shortServerIds = new HashSet<>(); + for (LwM2MBootstrapServerCredential bootstrapServerCredential : lwM2MBootstrapServersConfigurations) { + AbstractLwM2MBootstrapServerCredential serverConfig = (AbstractLwM2MBootstrapServerCredential) bootstrapServerCredential; + if (!isBootstrapServerUpdateEnable && serverConfig.isBootstrapServerIs()) { + throw new DeviceCredentialsValidationException("Bootstrap config must not include \"Bootstrap Server\". \"Include Bootstrap Server updates\" is " + isBootstrapServerUpdateEnable + "."); + } + String server = serverConfig.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server" + " shortServerId: " + serverConfig.getShortServerId() + ":"; + if (serverConfig.getShortServerId() < 1 || serverConfig.getShortServerId() > 65534) { + throw new DeviceCredentialsValidationException(server + " ShortServerId must not be less than 1 and more than 65534!"); + } + if (!shortServerIds.add(serverConfig.getShortServerId())) { + throw new DeviceCredentialsValidationException(server + " \"Short server Id\" value = " + serverConfig.getShortServerId() + ". This value must be a unique value for all servers!"); + } + String uri = serverConfig.getHost() + ":" + serverConfig.getPort(); + if (!uris.add(uri)) { + throw new DeviceCredentialsValidationException(server + " \"Host + port\" value = " + uri + ". This value must be a unique value for all servers!"); + } + Integer port; + if (LwM2MSecurityMode.NO_SEC.equals(serverConfig.getSecurityMode())) { + port = serverConfig.isBootstrapServerIs() ? 5687 : 5685; + } else { + port = serverConfig.isBootstrapServerIs() ? 5688 : 5686; + } + if (serverConfig.getPort() == null || serverConfig.getPort().intValue() != port) { + throw new DeviceCredentialsValidationException(server + " \"Port\" value = " + serverConfig.getPort() + ". This value for security " + serverConfig.getSecurityMode().name() + " must be " + port + "!"); + } } } - private void checkProtoFileCommonSettings(String schemaName, boolean isEmptySettings, String invalidSettingsMessage) { - if (!isEmptySettings) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + invalidSettingsMessage); + private void validateLwm2mServersCredentialOfBootstrapForClient(LwM2MBootstrapServerCredential bootstrapServerConfig) { + String server; + switch (bootstrapServerConfig.getSecurityMode()) { + case NO_SEC: + case PSK: + break; + case RPK: + RPKLwM2MBootstrapServerCredential rpkServerCredentials = (RPKLwM2MBootstrapServerCredential) bootstrapServerConfig; + server = rpkServerCredentials.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server"; + if (StringUtils.isEmpty(rpkServerCredentials.getServerPublicKey())) { + throw new DeviceCredentialsValidationException(server + " RPK public key must be specified!"); + } + try { + String pubkRpkSever = EncryptionUtil.pubkTrimNewLines(rpkServerCredentials.getServerPublicKey()); + rpkServerCredentials.setServerPublicKey(pubkRpkSever); + SecurityUtil.publicKey.decode(rpkServerCredentials.getDecodedCServerPublicKey()); + } catch (Exception e) { + throw new DeviceCredentialsValidationException(server + " RPK public key must be in standard [RFC7250] and then encoded to Base64 format!"); + } + break; + case X509: + X509LwM2MBootstrapServerCredential x509ServerCredentials = (X509LwM2MBootstrapServerCredential) bootstrapServerConfig; + server = x509ServerCredentials.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server"; + if (StringUtils.isEmpty(x509ServerCredentials.getServerPublicKey())) { + throw new DeviceCredentialsValidationException(server + " X509 certificate must be specified!"); + } + try { + String certServer = EncryptionUtil.certTrimNewLines(x509ServerCredentials.getServerPublicKey()); + x509ServerCredentials.setServerPublicKey(certServer); + SecurityUtil.certificate.decode(x509ServerCredentials.getDecodedCServerPublicKey()); + } catch (Exception e) { + throw new DeviceCredentialsValidationException(server + " X509 certificate must be in DER-encoded X509v3 format and support only EC algorithm and then encoded to Base64 format!"); + } + break; } } - private void checkTypeElements(String schemaName, ProtoFileElement protoFileElement) { - List types = protoFileElement.getTypes(); - if (!types.isEmpty()) { - if (types.stream().noneMatch(typeElement -> typeElement instanceof MessageElement)) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " At least one Message definition should exists!"); - } else { - checkEnumElements(schemaName, getEnumElements(types)); - checkMessageElements(schemaName, getMessageTypes(types)); - } + private void validateTelemetryDynamicMessageFields(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { + String deviceTelemetryProtoSchema = protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema(); + Descriptors.Descriptor telemetryDynamicMessageDescriptor = protoTransportPayloadTypeConfiguration.getTelemetryDynamicMessageDescriptor(deviceTelemetryProtoSchema); + if (telemetryDynamicMessageDescriptor == null) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get telemetryDynamicMessageDescriptor!"); } else { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Type elements is empty!"); + List fields = telemetryDynamicMessageDescriptor.getFields(); + if (CollectionUtils.isEmpty(fields)) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " " + telemetryDynamicMessageDescriptor.getName() + " fields is empty!"); + } else if (fields.size() == 2) { + Descriptors.FieldDescriptor tsFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("ts"); + Descriptors.FieldDescriptor valuesFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("values"); + if (tsFieldDescriptor != null && valuesFieldDescriptor != null) { + if (!Descriptors.FieldDescriptor.Type.MESSAGE.equals(valuesFieldDescriptor.getType())) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'values' has invalid data type. Only message type is supported!"); + } + if (!Descriptors.FieldDescriptor.Type.INT64.equals(tsFieldDescriptor.getType())) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'ts' has invalid data type. Only int64 type is supported!"); + } + if (!tsFieldDescriptor.hasOptionalKeyword()) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'ts' has invalid label. Field 'ts' should have optional keyword!"); + } + } + } } } - private void checkFieldElements(String schemaName, List fieldElements) { - if (!fieldElements.isEmpty()) { - boolean hasRequiredLabel = fieldElements.stream().anyMatch(fieldElement -> { - Field.Label label = fieldElement.getLabel(); - return label != null && label.equals(Field.Label.REQUIRED); - }); - if (hasRequiredLabel) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Required labels are not supported!"); + private void validateRpcRequestDynamicMessageFields(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { + DynamicMessage.Builder rpcRequestDynamicMessageBuilder = protoTransportPayloadTypeConfiguration.getRpcRequestDynamicMessageBuilder(protoTransportPayloadTypeConfiguration.getDeviceRpcRequestProtoSchema()); + Descriptors.Descriptor rpcRequestDynamicMessageDescriptor = rpcRequestDynamicMessageBuilder.getDescriptorForType(); + if (rpcRequestDynamicMessageDescriptor == null) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get rpcRequestDynamicMessageDescriptor!"); + } else { + if (CollectionUtils.isEmpty(rpcRequestDynamicMessageDescriptor.getFields()) || rpcRequestDynamicMessageDescriptor.getFields().size() != 3) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " " + rpcRequestDynamicMessageDescriptor.getName() + " message should always contains 3 fields: method, requestId and params!"); } - boolean hasDefaultValue = fieldElements.stream().anyMatch(fieldElement -> fieldElement.getDefaultValue() != null); - if (hasDefaultValue) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Default values are not supported!"); + Descriptors.FieldDescriptor methodFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("method"); + if (methodFieldDescriptor == null) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get field descriptor for field: method!"); + } else { + if (!Descriptors.FieldDescriptor.Type.STRING.equals(methodFieldDescriptor.getType())) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'method' has invalid data type. Only string type is supported!"); + } + if (methodFieldDescriptor.isRepeated()) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'method' has invalid label!"); + } } - } - } - - private void checkEnumElements(String schemaName, List enumTypes) { - if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getNestedTypes().isEmpty())) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Nested types in Enum definitions are not supported!"); - } - if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getOptions().isEmpty())) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Enum definitions options are not supported!"); - } - } - - private void checkMessageElements(String schemaName, List messageElementsList) { - if (!messageElementsList.isEmpty()) { - messageElementsList.forEach(messageElement -> { - checkProtoFileCommonSettings(schemaName, messageElement.getGroups().isEmpty(), - " Message definition groups don't support!"); - checkProtoFileCommonSettings(schemaName, messageElement.getOptions().isEmpty(), - " Message definition options don't support!"); - checkProtoFileCommonSettings(schemaName, messageElement.getExtensions().isEmpty(), - " Message definition extensions don't support!"); - checkProtoFileCommonSettings(schemaName, messageElement.getReserveds().isEmpty(), - " Message definition reserved elements don't support!"); - checkFieldElements(schemaName, messageElement.getFields()); - List oneOfs = messageElement.getOneOfs(); - if (!oneOfs.isEmpty()) { - oneOfs.forEach(oneOfElement -> { - checkProtoFileCommonSettings(schemaName, oneOfElement.getGroups().isEmpty(), - " OneOf definition groups don't support!"); - checkFieldElements(schemaName, oneOfElement.getFields()); - }); + Descriptors.FieldDescriptor requestIdFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("requestId"); + if (requestIdFieldDescriptor == null) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get field descriptor for field: requestId!"); + } else { + if (!Descriptors.FieldDescriptor.Type.INT32.equals(requestIdFieldDescriptor.getType())) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'requestId' has invalid data type. Only int32 type is supported!"); } - List nestedTypes = messageElement.getNestedTypes(); - if (!nestedTypes.isEmpty()) { - List nestedEnumTypes = getEnumElements(nestedTypes); - if (!nestedEnumTypes.isEmpty()) { - checkEnumElements(schemaName, nestedEnumTypes); - } - List nestedMessageTypes = getMessageTypes(nestedTypes); - checkMessageElements(schemaName, nestedMessageTypes); + if (requestIdFieldDescriptor.isRepeated()) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'requestId' has invalid label!"); } - }); + } + Descriptors.FieldDescriptor paramsFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("params"); + if (paramsFieldDescriptor == null) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get field descriptor for field: params!"); + } else { + if (paramsFieldDescriptor.isRepeated()) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'params' has invalid label!"); + } + } } } - - private List getMessageTypes(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof MessageElement) - .map(typeElement -> (MessageElement) typeElement) - .collect(Collectors.toList()); - } - - private List getEnumElements(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof EnumElement) - .map(typeElement -> (EnumElement) typeElement) - .collect(Collectors.toList()); - } - }; - private void validateTelemetryDynamicMessageFields(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { - String deviceTelemetryProtoSchema = protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema(); - Descriptors.Descriptor telemetryDynamicMessageDescriptor = protoTransportPayloadTypeConfiguration.getTelemetryDynamicMessageDescriptor(deviceTelemetryProtoSchema); - if (telemetryDynamicMessageDescriptor == null) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Failed to get telemetryDynamicMessageDescriptor!"); - } else { - List fields = telemetryDynamicMessageDescriptor.getFields(); - if (CollectionUtils.isEmpty(fields)) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " " + telemetryDynamicMessageDescriptor.getName() + " fields is empty!"); - } else if (fields.size() == 2) { - Descriptors.FieldDescriptor tsFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("ts"); - Descriptors.FieldDescriptor valuesFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("values"); - if (tsFieldDescriptor != null && valuesFieldDescriptor != null) { - if (!Descriptors.FieldDescriptor.Type.MESSAGE.equals(valuesFieldDescriptor.getType())) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'values' has invalid data type. Only message type is supported!"); - } - if (!Descriptors.FieldDescriptor.Type.INT64.equals(tsFieldDescriptor.getType())) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'ts' has invalid data type. Only int64 type is supported!"); - } - if (!tsFieldDescriptor.hasOptionalKeyword()) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'ts' has invalid label. Field 'ts' should have optional keyword!"); - } - } - } - } - } - - private void validateRpcRequestDynamicMessageFields(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { - DynamicMessage.Builder rpcRequestDynamicMessageBuilder = protoTransportPayloadTypeConfiguration.getRpcRequestDynamicMessageBuilder(protoTransportPayloadTypeConfiguration.getDeviceRpcRequestProtoSchema()); - Descriptors.Descriptor rpcRequestDynamicMessageDescriptor = rpcRequestDynamicMessageBuilder.getDescriptorForType(); - if (rpcRequestDynamicMessageDescriptor == null) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get rpcRequestDynamicMessageDescriptor!"); - } else { - if (CollectionUtils.isEmpty(rpcRequestDynamicMessageDescriptor.getFields()) || rpcRequestDynamicMessageDescriptor.getFields().size() != 3) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " " + rpcRequestDynamicMessageDescriptor.getName() + " message should always contains 3 fields: method, requestId and params!"); - } - Descriptors.FieldDescriptor methodFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("method"); - if (methodFieldDescriptor == null) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get field descriptor for field: method!"); - } else { - if (!Descriptors.FieldDescriptor.Type.STRING.equals(methodFieldDescriptor.getType())) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'method' has invalid data type. Only string type is supported!"); - } - if (methodFieldDescriptor.isRepeated()) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'method' has invalid label!"); - } - } - Descriptors.FieldDescriptor requestIdFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("requestId"); - if (requestIdFieldDescriptor == null) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get field descriptor for field: requestId!"); - } else { - if (!Descriptors.FieldDescriptor.Type.INT32.equals(requestIdFieldDescriptor.getType())) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'requestId' has invalid data type. Only int32 type is supported!"); - } - if (requestIdFieldDescriptor.isRepeated()) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'requestId' has invalid label!"); - } - } - Descriptors.FieldDescriptor paramsFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("params"); - if (paramsFieldDescriptor == null) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get field descriptor for field: params!"); - } else { - if (paramsFieldDescriptor.isRepeated()) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'params' has invalid label!"); - } - } - } - } - - private void validateLwm2mServersConfigOfBootstrapForClient(List lwM2MBootstrapServersConfigurations, boolean isBootstrapServerUpdateEnable) { - Set uris = new HashSet<>(); - Set shortServerIds = new HashSet<>(); - for (LwM2MBootstrapServerCredential bootstrapServerCredential : lwM2MBootstrapServersConfigurations) { - AbstractLwM2MBootstrapServerCredential serverConfig = (AbstractLwM2MBootstrapServerCredential) bootstrapServerCredential; - if (!isBootstrapServerUpdateEnable && serverConfig.isBootstrapServerIs()) { - throw new DeviceCredentialsValidationException("Bootstrap config must not include \"Bootstrap Server\". \"Include Bootstrap Server updates\" is " + isBootstrapServerUpdateEnable + "."); - } - String server = serverConfig.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server" + " shortServerId: " + serverConfig.getShortServerId() + ":"; - if (serverConfig.getShortServerId() < 1 || serverConfig.getShortServerId() > 65534) { - throw new DeviceCredentialsValidationException(server + " ShortServerId must not be less than 1 and more than 65534!"); - } - if (!shortServerIds.add(serverConfig.getShortServerId())) { - throw new DeviceCredentialsValidationException(server + " \"Short server Id\" value = " + serverConfig.getShortServerId() + ". This value must be a unique value for all servers!"); - } - String uri = serverConfig.getHost() + ":" + serverConfig.getPort(); - if (!uris.add(uri)) { - throw new DeviceCredentialsValidationException(server + " \"Host + port\" value = " + uri + ". This value must be a unique value for all servers!"); - } - Integer port; - if (LwM2MSecurityMode.NO_SEC.equals(serverConfig.getSecurityMode())) { - port = serverConfig.isBootstrapServerIs() ? 5687 : 5685; - } else { - port = serverConfig.isBootstrapServerIs() ? 5688 : 5686; - } - if (serverConfig.getPort() == null || serverConfig.getPort().intValue() != port) { - throw new DeviceCredentialsValidationException(server + " \"Port\" value = " + serverConfig.getPort() + ". This value for security " + serverConfig.getSecurityMode().name() + " must be " + port + "!"); - } - } - } - - private void validateLwm2mServersCredentialOfBootstrapForClient(LwM2MBootstrapServerCredential bootstrapServerConfig) { - String server; - switch (bootstrapServerConfig.getSecurityMode()) { - case NO_SEC: - case PSK: - break; - case RPK: - RPKLwM2MBootstrapServerCredential rpkServerCredentials = (RPKLwM2MBootstrapServerCredential) bootstrapServerConfig; - server = rpkServerCredentials.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server"; - if (StringUtils.isEmpty(rpkServerCredentials.getServerPublicKey())) { - throw new DeviceCredentialsValidationException(server + " RPK public key must be specified!"); - } - try { - String pubkRpkSever = EncryptionUtil.pubkTrimNewLines(rpkServerCredentials.getServerPublicKey()); - rpkServerCredentials.setServerPublicKey(pubkRpkSever); - SecurityUtil.publicKey.decode(rpkServerCredentials.getDecodedCServerPublicKey()); - } catch (Exception e) { - throw new DeviceCredentialsValidationException(server + " RPK public key must be in standard [RFC7250] and then encoded to Base64 format!"); - } - break; - case X509: - X509LwM2MBootstrapServerCredential x509ServerCredentials = (X509LwM2MBootstrapServerCredential) bootstrapServerConfig; - server = x509ServerCredentials.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server"; - if (StringUtils.isEmpty(x509ServerCredentials.getServerPublicKey())) { - throw new DeviceCredentialsValidationException(server + " X509 certificate must be specified!"); - } - - try { - String certServer = EncryptionUtil.certTrimNewLines(x509ServerCredentials.getServerPublicKey()); - x509ServerCredentials.setServerPublicKey(certServer); - SecurityUtil.certificate.decode(x509ServerCredentials.getDecodedCServerPublicKey()); - } catch (Exception e) { - throw new DeviceCredentialsValidationException(server + " X509 certificate must be in DER-encoded X509v3 format and support only EC algorithm and then encoded to Base64 format!"); - } - break; - } - } - private PaginatedRemover tenantDeviceProfilesRemover = new PaginatedRemover() { From 163d100173c866dbd48c6c5914c68f48e1b0eb3b Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Mon, 28 Feb 2022 11:39:12 +0200 Subject: [PATCH 02/26] fix typos after merge with master --- .../service/validator/DeviceProfileDataValidator.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index fd1a02d628..7de8b4ed84 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -17,16 +17,6 @@ package org.thingsboard.server.dao.service.validator; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import com.squareup.wire.Syntax; -import com.squareup.wire.schema.Field; -import com.squareup.wire.schema.Location; -import com.squareup.wire.schema.internal.parser.EnumElement; -import com.squareup.wire.schema.internal.parser.FieldElement; -import com.squareup.wire.schema.internal.parser.MessageElement; -import com.squareup.wire.schema.internal.parser.OneOfElement; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; -import com.squareup.wire.schema.internal.parser.ProtoParser; -import com.squareup.wire.schema.internal.parser.TypeElement; import org.eclipse.leshan.core.util.SecurityUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; From 430c0784f863e51be340657f1a548796d1720aad Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Mon, 28 Feb 2022 11:45:35 +0200 Subject: [PATCH 03/26] fix license header in MqttTransportHandler.java --- .../thingsboard/server/transport/mqtt/MqttTransportHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 1e7c2f086a..e80f735d89 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From 58302fc92741286b2bdb7cba23e95bf884ed8dbd Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Thu, 16 Jun 2022 15:35:31 +0300 Subject: [PATCH 04/26] fix typo in BaseDeviceProfileControllerTest after merge --- .../server/controller/BaseDeviceProfileControllerTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java index 8b3a61e9ff..8d088625d5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java @@ -694,8 +694,6 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController "}", "[Transport Configuration] invalid rpc request proto schema provided! Failed to get field descriptor for field: params!"); } -<<<<<<< HEAD -======= @Test public void testSaveDeviceProfileWithSendAckOnValidationException() throws Exception { JsonTransportPayloadConfiguration jsonTransportPayloadConfiguration = new JsonTransportPayloadConfiguration(); @@ -722,7 +720,6 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController return savedDeviceProfile; } ->>>>>>> develop/3.4 private void testSaveDeviceProfileWithInvalidProtoSchema(String schema, String errorMsg) throws Exception { ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema, null, null); MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration, false); From 28cf298d83d8eb9344ab8b1e586f7a3511c1e475 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Tue, 19 Jul 2022 14:45:35 +0300 Subject: [PATCH 05/26] fix typos after merge in coap tests --- .../AbstractCoapAttributesIntegrationTest.java | 10 +++++----- .../CoapAttributesRequestProtoIntegrationTest.java | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java index b0b5c34d5a..7617f2b826 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java @@ -157,11 +157,11 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType(); assertNotNull(postAttributesMsgDescriptor); DynamicMessage postAttributesMsg = postAttributesBuilder - .setField(postAttributesMsgDescriptor.findFieldByName("attribute1"), "value1") - .setField(postAttributesMsgDescriptor.findFieldByName("attribute2"), true) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute3"), 42.0) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute4"), 73) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute5"), jsonObject) + .setField(postAttributesMsgDescriptor.findFieldByName("clientStr"), "value1") + .setField(postAttributesMsgDescriptor.findFieldByName("clientBool"), true) + .setField(postAttributesMsgDescriptor.findFieldByName("clientDbl"), 42.0) + .setField(postAttributesMsgDescriptor.findFieldByName("clientLong"), 73) + .setField(postAttributesMsgDescriptor.findFieldByName("clientJson"), jsonObject) .build(); return postAttributesMsg.toByteArray(); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java index 9244cd5b1f..5f300ba68e 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java @@ -43,5 +43,4 @@ public class CoapAttributesRequestProtoIntegrationTest extends CoapAttributesReq public void testRequestAttributesValuesFromTheServer() throws Exception { processProtoTestRequestAttributesValuesFromTheServer(); } - } From ecd2be94565a041ed032d0a0b72c3f6e2f931c2d Mon Sep 17 00:00:00 2001 From: devaskim Date: Wed, 14 Sep 2022 16:32:52 +0500 Subject: [PATCH 06/26] Add API to create telemetry websocket updater in custom widgets. --- ui-ngx/src/app/core/services/utils.service.ts | 35 +++++++++++++++++++ .../widget/dynamic-widget.component.ts | 2 ++ .../app/modules/home/models/services.map.ts | 4 ++- .../home/models/widget-component.models.ts | 4 +++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/core/services/utils.service.ts b/ui-ngx/src/app/core/services/utils.service.ts index b901b87328..1c62f5e7bf 100644 --- a/ui-ngx/src/app/core/services/utils.service.ts +++ b/ui-ngx/src/app/core/services/utils.service.ts @@ -43,6 +43,15 @@ import { WidgetInfo } from '@home/models/widget-component.models'; import jsonSchemaDefaults from 'json-schema-defaults'; import materialIconsCodepoints from '!raw-loader!./material-icons-codepoints.raw'; import { Observable, of, ReplaySubject } from 'rxjs'; +import { publishReplay, refCount } from 'rxjs/operators'; +import { WidgetContext } from '@app/modules/home/models/widget-component.models'; +import { + AttributeData, + TelemetrySubscriber, + TelemetryType, + LatestTelemetry +} from '@shared/models/telemetry/telemetry.models'; +import { EntityId } from '@shared/models/id/entity-id'; const i18nRegExp = new RegExp(`{${i18nPrefix}:[^{}]+}`, 'g'); @@ -479,4 +488,30 @@ export class UtilsService { return defaultValue; } } + + public getEntityIdFromDatasource(dataSource: Datasource) { + return { id: dataSource.entityId, entityType: dataSource.entityType}; + } + + public createTelemetrySubscriber(ctx: WidgetContext, + entityId?: EntityId, + type: TelemetryType = LatestTelemetry.LATEST_TELEMETRY, + keys: string[] = null): TelemetrySubscriber { + if (!entityId && ctx.datasources.length > 0) { + entityId = this.getEntityIdFromDatasource(ctx.datasources[0]); + } + return TelemetrySubscriber.createEntityAttributesSubscription(ctx.telemetryWsService, entityId, type, ctx.ngZone, keys); + } + + public subscribeToEntityTelemetry(ctx: WidgetContext, + entityId?: EntityId, + type: TelemetryType = LatestTelemetry.LATEST_TELEMETRY, + keys: string[] = null): Observable> { + ctx.telemetrySubscriber = this.createTelemetrySubscriber(ctx, entityId, type, keys); + ctx.telemetrySubscriber.subscribe(); + return ctx.telemetrySubscriber.attributeData$().pipe( + publishReplay(1), + refCount() + ); + } } diff --git a/ui-ngx/src/app/modules/home/components/widget/dynamic-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/dynamic-widget.component.ts index 59b1959269..2aea3096bb 100644 --- a/ui-ngx/src/app/modules/home/components/widget/dynamic-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/dynamic-widget.component.ts @@ -40,6 +40,7 @@ import { AuthService } from '@core/auth/auth.service'; import { DialogService } from '@core/services/dialog.service'; import { CustomDialogService } from '@home/components/widget/dialog/custom-dialog.service'; import { ResourceService } from '@core/http/resource.service'; +import { TelemetryWebsocketService } from '@core/ws/telemetry-websocket.service'; import { DatePipe } from '@angular/common'; import { TranslateService } from '@ngx-translate/core'; import { DomSanitizer } from '@angular/platform-browser'; @@ -80,6 +81,7 @@ export class DynamicWidgetComponent extends PageComponent implements IDynamicWid this.ctx.dialogs = $injector.get(DialogService); this.ctx.customDialog = $injector.get(CustomDialogService); this.ctx.resourceService = $injector.get(ResourceService); + this.ctx.telemetryWsService = $injector.get(TelemetryWebsocketService); this.ctx.date = $injector.get(DatePipe); this.ctx.translate = $injector.get(TranslateService); this.ctx.http = $injector.get(HttpClient); diff --git a/ui-ngx/src/app/modules/home/models/services.map.ts b/ui-ngx/src/app/modules/home/models/services.map.ts index 93ec69da61..7514e972b5 100644 --- a/ui-ngx/src/app/modules/home/models/services.map.ts +++ b/ui-ngx/src/app/modules/home/models/services.map.ts @@ -39,6 +39,7 @@ import { OtaPackageService } from '@core/http/ota-package.service'; import { AuthService } from '@core/auth/auth.service'; import { ResourceService } from '@core/http/resource.service'; import { TwoFactorAuthenticationService } from '@core/http/two-factor-authentication.service'; +import { TelemetryWebsocketService } from '@core/ws/telemetry-websocket.service'; export const ServicesMap = new Map>( [ @@ -65,6 +66,7 @@ export const ServicesMap = new Map>( ['otaPackageService', OtaPackageService], ['authService', AuthService], ['resourceService', ResourceService], - ['twoFactorAuthenticationService', TwoFactorAuthenticationService] + ['twoFactorAuthenticationService', TwoFactorAuthenticationService], + ['telemetryWsService', TelemetryWebsocketService] ] ); diff --git a/ui-ngx/src/app/modules/home/models/widget-component.models.ts b/ui-ngx/src/app/modules/home/models/widget-component.models.ts index 4686fe88e6..6bb5be6cc3 100644 --- a/ui-ngx/src/app/modules/home/models/widget-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/widget-component.models.ts @@ -73,6 +73,7 @@ import { DialogService } from '@core/services/dialog.service'; import { CustomDialogService } from '@home/components/widget/dialog/custom-dialog.service'; import { AuthService } from '@core/auth/auth.service'; import { ResourceService } from '@core/http/resource.service'; +import { TelemetryWebsocketService } from '@core/ws/telemetry-websocket.service'; import { DatePipe } from '@angular/common'; import { TranslateService } from '@ngx-translate/core'; import { PageLink } from '@shared/models/page/page-link'; @@ -84,6 +85,7 @@ import * as RxJS from 'rxjs'; import * as RxJSOperators from 'rxjs/operators'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { EntityId } from '@shared/models/id/entity-id'; +import { TelemetrySubscriber } from '@app/shared/public-api'; export interface IWidgetAction { name: string; @@ -173,6 +175,8 @@ export class WidgetContext { dialogs: DialogService; customDialog: CustomDialogService; resourceService: ResourceService; + telemetryWsService: TelemetryWebsocketService; + telemetrySubscriber?: TelemetrySubscriber; date: DatePipe; translate: TranslateService; http: HttpClient; From 287c1fd48a882b9c5ec04ac1145d42b6f82db86a Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Wed, 5 Oct 2022 10:09:01 +0300 Subject: [PATCH 07/26] add to msgData --- .../metadata/TbAbstractGetAttributesNode.java | 64 +++++++++++++++---- .../engine/metadata/TbGetAttributesNode.java | 6 +- .../TbGetAttributesNodeConfiguration.java | 2 + .../TbGetDeviceAttrNodeConfiguration.java | 1 + 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index b2c029589c..6f90078cdb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -17,6 +17,7 @@ package org.thingsboard.rule.engine.metadata; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.json.JsonWriteFeature; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; @@ -25,6 +26,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.JsonParseException; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; @@ -57,10 +59,12 @@ public abstract class TbAbstractGetAttributesNode> failuresMap = new ConcurrentHashMap<>(); ListenableFuture> allFutures = Futures.allAsList( - putLatestTelemetry(ctx, entityId, msg, LATEST_TS, TbNodeUtils.processPatterns(config.getLatestTsKeyNames(), msg), failuresMap), - putAttrAsync(ctx, entityId, msg, CLIENT_SCOPE, TbNodeUtils.processPatterns(config.getClientAttributeNames(), msg), failuresMap, "cs_"), - putAttrAsync(ctx, entityId, msg, SHARED_SCOPE, TbNodeUtils.processPatterns(config.getSharedAttributeNames(), msg), failuresMap, "shared_"), - putAttrAsync(ctx, entityId, msg, SERVER_SCOPE, TbNodeUtils.processPatterns(config.getServerAttributeNames(), msg), failuresMap, "ss_") + putLatestTelemetry(ctx, entityId, msg, LATEST_TS, TbNodeUtils.processPatterns(config.getLatestTsKeyNames(), msg), failuresMap, msgNewData), + putAttrAsync(ctx, entityId, msg, CLIENT_SCOPE, TbNodeUtils.processPatterns(config.getClientAttributeNames(), msg), failuresMap, "cs_", msgNewData), + putAttrAsync(ctx, entityId, msg, SHARED_SCOPE, TbNodeUtils.processPatterns(config.getSharedAttributeNames(), msg), failuresMap, "shared_", msgNewData), + putAttrAsync(ctx, entityId, msg, SERVER_SCOPE, TbNodeUtils.processPatterns(config.getServerAttributeNames(), msg), failuresMap, "ss_", msgNewData) ); withCallback(allFutures, i -> { if (!failuresMap.isEmpty()) { throw reportFailures(failuresMap); } - ctx.tellSuccess(msg); + if (fetchToData) { + ObjectNode msgDataNode = (ObjectNode) JacksonUtil.toJsonNode(msg.getData()); + msgDataNode.setAll(msgNewData); + ctx.tellSuccess(TbMsg.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), JacksonUtil.toString(msgDataNode))); + } else { + ctx.tellSuccess(msg); + } }, t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } - private ListenableFuture putAttrAsync(TbContext ctx, EntityId entityId, TbMsg msg, String scope, List keys, ConcurrentHashMap> failuresMap, String prefix) { + private ListenableFuture putAttrAsync(TbContext ctx, EntityId entityId, TbMsg msg, String scope, List keys, ConcurrentHashMap> failuresMap, String prefix, ObjectNode msgData) { if (CollectionUtils.isEmpty(keys)) { return Futures.immediateFuture(null); } @@ -109,7 +127,13 @@ public abstract class TbAbstractGetAttributesNode { if (!CollectionUtils.isEmpty(attributeKvEntryList)) { List existingAttributesKvEntry = attributeKvEntryList.stream().filter(attributeKvEntry -> keys.contains(attributeKvEntry.getKey())).collect(Collectors.toList()); - existingAttributesKvEntry.forEach(kvEntry -> msg.getMetaData().putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString())); + existingAttributesKvEntry.forEach(kvEntry -> { + if (fetchToData) { + msgData.put(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); + } else { + msg.getMetaData().putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); + } + }); if (existingAttributesKvEntry.size() != keys.size() && BooleanUtils.toBooleanDefaultIfNull(this.config.isTellFailureIfAbsent(), true)) { getNotExistingKeys(existingAttributesKvEntry, keys).forEach(key -> computeFailuresMap(scope, failuresMap, key)); } @@ -122,7 +146,7 @@ public abstract class TbAbstractGetAttributesNode putLatestTelemetry(TbContext ctx, EntityId entityId, TbMsg msg, String scope, List keys, ConcurrentHashMap> failuresMap) { + private ListenableFuture putLatestTelemetry(TbContext ctx, EntityId entityId, TbMsg msg, String scope, List keys, ConcurrentHashMap> failuresMap, ObjectNode msgData) { if (CollectionUtils.isEmpty(keys)) { return Futures.immediateFuture(null); } @@ -134,16 +158,24 @@ public abstract class TbAbstractGetAttributesNode getNotExistingKeys(List existingAttributesKvEntry, List allKeys) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNode.java index 431fe64c09..6388f9c44e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNode.java @@ -34,9 +34,9 @@ import org.thingsboard.server.common.msg.TbMsg; @RuleNode(type = ComponentType.ENRICHMENT, name = "originator attributes", configClazz = TbGetAttributesNodeConfiguration.class, - nodeDescription = "Add Message Originator Attributes or Latest Telemetry into Message Metadata", - nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message metadata " + - "with specific prefix: cs/shared/ss. Latest telemetry value added into metadata without prefix. " + + nodeDescription = "Add Message Originator Attributes or Latest Telemetry into Message Metadata/data", + nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message metadata or data " + + "with specific prefix: cs/shared/ss. Latest telemetry value added into metadata/data without prefix. " + "To access those attributes in other nodes this template can be used " + "metadata.cs_temperature or metadata.shared_limit ", uiResources = {"static/rulenode/rulenode-core-config.js"}, diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeConfiguration.java index 4fc892296f..67766e5b51 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeConfiguration.java @@ -35,6 +35,7 @@ public class TbGetAttributesNodeConfiguration implements NodeConfiguration Date: Fri, 7 Oct 2022 10:27:58 +0300 Subject: [PATCH 08/26] refactor node --- .../thingsboard/common/util/JacksonUtil.java | 22 +++ .../metadata/TbAbstractGetAttributesNode.java | 146 +++++++++--------- 2 files changed, 98 insertions(+), 70 deletions(-) diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index 9c4b68d3c0..46d1881ab7 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -24,6 +24,8 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.KvEntry; import java.io.IOException; import java.util.ArrayList; @@ -212,4 +214,24 @@ public class JacksonUtil { } } } + + public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry, String prefix) { + if (kvEntry.getDataType() == DataType.BOOLEAN) { + kvEntry.getBooleanValue().ifPresent(value -> entityNode.put(prefix + kvEntry.getKey(), value)); + } else if (kvEntry.getDataType() == DataType.DOUBLE) { + kvEntry.getDoubleValue().ifPresent(value -> entityNode.put(prefix + kvEntry.getKey(), value)); + } else if (kvEntry.getDataType() == DataType.LONG) { + kvEntry.getLongValue().ifPresent(value -> entityNode.put(prefix + kvEntry.getKey(), value)); + } else if (kvEntry.getDataType() == DataType.JSON) { + if (kvEntry.getJsonValue().isPresent()) { + entityNode.set(prefix + kvEntry.getKey(), JacksonUtil.valueToTree(kvEntry.getJsonValue().get())); + } + } else { + entityNode.put(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); + } + } + + public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry) { + addKvEntry(entityNode, kvEntry, ""); + } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 6f90078cdb..89c92305ca 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -34,13 +34,19 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -60,11 +66,15 @@ public abstract class TbAbstractGetAttributesNode> failuresMap = new ConcurrentHashMap<>(); - ListenableFuture> allFutures = Futures.allAsList( - putLatestTelemetry(ctx, entityId, msg, LATEST_TS, TbNodeUtils.processPatterns(config.getLatestTsKeyNames(), msg), failuresMap, msgNewData), - putAttrAsync(ctx, entityId, msg, CLIENT_SCOPE, TbNodeUtils.processPatterns(config.getClientAttributeNames(), msg), failuresMap, "cs_", msgNewData), - putAttrAsync(ctx, entityId, msg, SHARED_SCOPE, TbNodeUtils.processPatterns(config.getSharedAttributeNames(), msg), failuresMap, "shared_", msgNewData), - putAttrAsync(ctx, entityId, msg, SERVER_SCOPE, TbNodeUtils.processPatterns(config.getServerAttributeNames(), msg), failuresMap, "ss_", msgNewData) + ListenableFuture>>> allFutures = Futures.allAsList( + getLatestTelemetry(ctx, entityId, msg, LATEST_TS, TbNodeUtils.processPatterns(config.getLatestTsKeyNames(), msg), failuresMap), + getAttrAsync(ctx, entityId, CLIENT_SCOPE, TbNodeUtils.processPatterns(config.getClientAttributeNames(), msg), failuresMap), + getAttrAsync(ctx, entityId, SHARED_SCOPE, TbNodeUtils.processPatterns(config.getSharedAttributeNames(), msg), failuresMap), + getAttrAsync(ctx, entityId, SERVER_SCOPE, TbNodeUtils.processPatterns(config.getServerAttributeNames(), msg), failuresMap) ); - withCallback(allFutures, i -> { + withCallback(allFutures, futuresList -> { if (!failuresMap.isEmpty()) { throw reportFailures(failuresMap); } + TbMsgMetaData msgMetaData = msg.getMetaData(); + futuresList.stream().filter(Objects::nonNull).forEach(kvEntriesMap -> { + kvEntriesMap.forEach((keyScope, kvEntryList) -> { + String prefix = getPrefix(keyScope); + kvEntryList.forEach(kvEntry -> { + if (fetchToData) { + JacksonUtil.addKvEntry((ObjectNode) msgDataNode, kvEntry, prefix); + } else { + msgMetaData.putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); + } + }); + }); + }); if (fetchToData) { - ObjectNode msgDataNode = (ObjectNode) JacksonUtil.toJsonNode(msg.getData()); - msgDataNode.setAll(msgNewData); ctx.tellSuccess(TbMsg.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), JacksonUtil.toString(msgDataNode))); } else { ctx.tellSuccess(msg); @@ -119,100 +139,86 @@ public abstract class TbAbstractGetAttributesNode ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } - private ListenableFuture putAttrAsync(TbContext ctx, EntityId entityId, TbMsg msg, String scope, List keys, ConcurrentHashMap> failuresMap, String prefix, ObjectNode msgData) { + private ListenableFuture>> getAttrAsync(TbContext ctx, EntityId entityId, String scope, List keys, ConcurrentHashMap> failuresMap) { if (CollectionUtils.isEmpty(keys)) { return Futures.immediateFuture(null); } ListenableFuture> attributeKvEntryListFuture = ctx.getAttributesService().find(ctx.getTenantId(), entityId, scope, keys); return Futures.transform(attributeKvEntryListFuture, attributeKvEntryList -> { - if (!CollectionUtils.isEmpty(attributeKvEntryList)) { - List existingAttributesKvEntry = attributeKvEntryList.stream().filter(attributeKvEntry -> keys.contains(attributeKvEntry.getKey())).collect(Collectors.toList()); - existingAttributesKvEntry.forEach(kvEntry -> { - if (fetchToData) { - msgData.put(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); - } else { - msg.getMetaData().putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); - } - }); - if (existingAttributesKvEntry.size() != keys.size() && BooleanUtils.toBooleanDefaultIfNull(this.config.isTellFailureIfAbsent(), true)) { - getNotExistingKeys(existingAttributesKvEntry, keys).forEach(key -> computeFailuresMap(scope, failuresMap, key)); - } - } else { - if (BooleanUtils.toBooleanDefaultIfNull(this.config.isTellFailureIfAbsent(), true)) { - keys.forEach(key -> computeFailuresMap(scope, failuresMap, key)); - } + if (isTellFailureIfAbsent && attributeKvEntryList.size() != keys.size()) { + getNotExistingKeys(attributeKvEntryList, keys).forEach(key -> computeFailuresMap(scope, failuresMap, key)); } - return null; + Map> mapAttributeKvEntry = new HashMap<>(); + mapAttributeKvEntry.put(scope, attributeKvEntryList); + return mapAttributeKvEntry; }, MoreExecutors.directExecutor()); } - private ListenableFuture putLatestTelemetry(TbContext ctx, EntityId entityId, TbMsg msg, String scope, List keys, ConcurrentHashMap> failuresMap, ObjectNode msgData) { + private ListenableFuture>> getLatestTelemetry(TbContext ctx, EntityId entityId, TbMsg msg, String scope, List keys, ConcurrentHashMap> failuresMap) { if (CollectionUtils.isEmpty(keys)) { return Futures.immediateFuture(null); } - ListenableFuture> latest = ctx.getTimeseriesService().findLatest(ctx.getTenantId(), entityId, keys); - return Futures.transform(latest, l -> { - l.forEach(r -> { - boolean getLatestValueWithTs = BooleanUtils.toBooleanDefaultIfNull(this.config.isGetLatestValueWithTs(), false); - if (BooleanUtils.toBooleanDefaultIfNull(this.config.isTellFailureIfAbsent(), true)) { - if (r.getValue() == null) { - computeFailuresMap(scope, failuresMap, r.getKey()); - } else if (getLatestValueWithTs) { - putValueWithTs(msg, r, msgData); - } else { - if (fetchToData) { - msgData.put(r.getKey(), r.getValueAsString()); - } else { - msg.getMetaData().putValue(r.getKey(), r.getValueAsString()); - } + ListenableFuture> latestTelemetryFutures = ctx.getTimeseriesService().findLatest(ctx.getTenantId(), entityId, keys); + return Futures.transform(latestTelemetryFutures, tsKvEntries -> { + List listTsKvEntry = new ArrayList<>(); + tsKvEntries.forEach(tsKvEntry -> { + if (tsKvEntry.getValue() == null) { + if (isTellFailureIfAbsent) { + computeFailuresMap(scope, failuresMap, tsKvEntry.getKey()); } + } else if (getLatestValueWithTs) { + listTsKvEntry.add(getValueWithTs(tsKvEntry)); } else { - if (r.getValue() != null) { - if (getLatestValueWithTs) { - putValueWithTs(msg, r, msgData); - } else { - if (fetchToData) { - msgData.put(r.getKey(), r.getValueAsString()); - } else { - msg.getMetaData().putValue(r.getKey(), r.getValueAsString()); - } - } - } + listTsKvEntry.add(new BasicTsKvEntry(tsKvEntry.getTs(), tsKvEntry)); } }); - return null; + Map> mapTsKvEntry = new HashMap<>(); + mapTsKvEntry.put(scope, listTsKvEntry); + return mapTsKvEntry; }, MoreExecutors.directExecutor()); } - private void putValueWithTs(TbMsg msg, TsKvEntry r, ObjectNode msgData) { - ObjectNode value = mapper.createObjectNode(); - value.put(TS, r.getTs()); - switch (r.getDataType()) { + private TsKvEntry getValueWithTs(TsKvEntry tsKvEntry) { + ObjectNode value = JacksonUtil.newObjectNode(); + value.put(TS, tsKvEntry.getTs()); + switch (tsKvEntry.getDataType()) { case STRING: - value.put(VALUE, r.getValueAsString()); + value.put(VALUE, tsKvEntry.getValueAsString()); break; case LONG: - value.put(VALUE, r.getLongValue().get()); + value.put(VALUE, tsKvEntry.getLongValue().get()); break; case BOOLEAN: - value.put(VALUE, r.getBooleanValue().get()); + value.put(VALUE, tsKvEntry.getBooleanValue().get()); break; case DOUBLE: - value.put(VALUE, r.getDoubleValue().get()); + value.put(VALUE, tsKvEntry.getDoubleValue().get()); break; case JSON: try { - value.set(VALUE, mapper.readTree(r.getJsonValue().get())); + value.set(VALUE, mapper.readTree(tsKvEntry.getJsonValue().get())); } catch (IOException e) { - throw new JsonParseException("Can't parse jsonValue: " + r.getJsonValue().get(), e); + throw new JsonParseException("Can't parse jsonValue: " + tsKvEntry.getJsonValue().get(), e); } break; } - if (fetchToData) { - msgData.putIfAbsent(r.getKey(), value); - } else { - msg.getMetaData().putValue(r.getKey(), value.toString()); + return new BasicTsKvEntry(tsKvEntry.getTs(), new JsonDataEntry(tsKvEntry.getKey(), value.toString())); + } + + private String getPrefix(String scope) { + String prefix = ""; + switch (scope) { + case CLIENT_SCOPE: + prefix = "cs_"; + break; + case SHARED_SCOPE: + prefix = "shared_"; + break; + case SERVER_SCOPE: + prefix = "ss_"; + break; } + return prefix; } private List getNotExistingKeys(List existingAttributesKvEntry, List allKeys) { From 0ec08b8448d75fd85ce10ff9d1b0c00156888c0b Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Fri, 7 Oct 2022 12:13:48 +0300 Subject: [PATCH 09/26] code review --- .../thingsboard/common/util/JacksonUtil.java | 14 +++--- .../metadata/TbAbstractGetAttributesNode.java | 49 ++++--------------- 2 files changed, 16 insertions(+), 47 deletions(-) diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index 46d1881ab7..1bfd85351c 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -215,23 +215,23 @@ public class JacksonUtil { } } - public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry, String prefix) { + public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry, String key) { if (kvEntry.getDataType() == DataType.BOOLEAN) { - kvEntry.getBooleanValue().ifPresent(value -> entityNode.put(prefix + kvEntry.getKey(), value)); + kvEntry.getBooleanValue().ifPresent(value -> entityNode.put(key, value)); } else if (kvEntry.getDataType() == DataType.DOUBLE) { - kvEntry.getDoubleValue().ifPresent(value -> entityNode.put(prefix + kvEntry.getKey(), value)); + kvEntry.getDoubleValue().ifPresent(value -> entityNode.put(key, value)); } else if (kvEntry.getDataType() == DataType.LONG) { - kvEntry.getLongValue().ifPresent(value -> entityNode.put(prefix + kvEntry.getKey(), value)); + kvEntry.getLongValue().ifPresent(value -> entityNode.put(key, value)); } else if (kvEntry.getDataType() == DataType.JSON) { if (kvEntry.getJsonValue().isPresent()) { - entityNode.set(prefix + kvEntry.getKey(), JacksonUtil.valueToTree(kvEntry.getJsonValue().get())); + entityNode.set(key, JacksonUtil.valueToTree(kvEntry.getJsonValue().get())); } } else { - entityNode.put(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); + entityNode.put(key, kvEntry.getValueAsString()); } } public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry) { - addKvEntry(entityNode, kvEntry, ""); + addKvEntry(entityNode, kvEntry, kvEntry.getKey()); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 89c92305ca..c516811e9c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -15,15 +15,11 @@ */ package org.thingsboard.rule.engine.metadata; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.json.JsonWriteFeature; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.JsonParseException; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.thingsboard.common.util.JacksonUtil; @@ -39,9 +35,7 @@ import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -59,8 +53,6 @@ import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; public abstract class TbAbstractGetAttributesNode implements TbNode { - private static ObjectMapper mapper = new ObjectMapper(); - private static final String VALUE = "value"; private static final String TS = "ts"; @@ -73,10 +65,8 @@ public abstract class TbAbstractGetAttributesNode> failuresMap = new ConcurrentHashMap<>(); - ListenableFuture>>> allFutures = Futures.allAsList( + ListenableFuture>>> allFutures = Futures.allAsList( getLatestTelemetry(ctx, entityId, msg, LATEST_TS, TbNodeUtils.processPatterns(config.getLatestTsKeyNames(), msg), failuresMap), getAttrAsync(ctx, entityId, CLIENT_SCOPE, TbNodeUtils.processPatterns(config.getClientAttributeNames(), msg), failuresMap), getAttrAsync(ctx, entityId, SHARED_SCOPE, TbNodeUtils.processPatterns(config.getSharedAttributeNames(), msg), failuresMap), @@ -118,15 +108,14 @@ public abstract class TbAbstractGetAttributesNode { kvEntriesMap.forEach((keyScope, kvEntryList) -> { String prefix = getPrefix(keyScope); kvEntryList.forEach(kvEntry -> { if (fetchToData) { - JacksonUtil.addKvEntry((ObjectNode) msgDataNode, kvEntry, prefix); + JacksonUtil.addKvEntry((ObjectNode) msgDataNode, kvEntry, prefix + kvEntry.getKey()); } else { - msgMetaData.putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); + msg.getMetaData().putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); } }); }); @@ -139,7 +128,7 @@ public abstract class TbAbstractGetAttributesNode ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } - private ListenableFuture>> getAttrAsync(TbContext ctx, EntityId entityId, String scope, List keys, ConcurrentHashMap> failuresMap) { + private ListenableFuture>> getAttrAsync(TbContext ctx, EntityId entityId, String scope, List keys, ConcurrentHashMap> failuresMap) { if (CollectionUtils.isEmpty(keys)) { return Futures.immediateFuture(null); } @@ -148,13 +137,13 @@ public abstract class TbAbstractGetAttributesNode computeFailuresMap(scope, failuresMap, key)); } - Map> mapAttributeKvEntry = new HashMap<>(); + Map> mapAttributeKvEntry = new HashMap<>(); mapAttributeKvEntry.put(scope, attributeKvEntryList); return mapAttributeKvEntry; }, MoreExecutors.directExecutor()); } - private ListenableFuture>> getLatestTelemetry(TbContext ctx, EntityId entityId, TbMsg msg, String scope, List keys, ConcurrentHashMap> failuresMap) { + private ListenableFuture>> getLatestTelemetry(TbContext ctx, EntityId entityId, TbMsg msg, String scope, List keys, ConcurrentHashMap> failuresMap) { if (CollectionUtils.isEmpty(keys)) { return Futures.immediateFuture(null); } @@ -172,7 +161,7 @@ public abstract class TbAbstractGetAttributesNode> mapTsKvEntry = new HashMap<>(); + Map> mapTsKvEntry = new HashMap<>(); mapTsKvEntry.put(scope, listTsKvEntry); return mapTsKvEntry; }, MoreExecutors.directExecutor()); @@ -181,27 +170,7 @@ public abstract class TbAbstractGetAttributesNode Date: Fri, 7 Oct 2022 13:07:12 +0300 Subject: [PATCH 10/26] add JacksonUtil.toJsonNode --- .../src/main/java/org/thingsboard/common/util/JacksonUtil.java | 2 +- .../rule/engine/metadata/TbAbstractGetAttributesNode.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index 1bfd85351c..55d3c254d7 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -224,7 +224,7 @@ public class JacksonUtil { kvEntry.getLongValue().ifPresent(value -> entityNode.put(key, value)); } else if (kvEntry.getDataType() == DataType.JSON) { if (kvEntry.getJsonValue().isPresent()) { - entityNode.set(key, JacksonUtil.valueToTree(kvEntry.getJsonValue().get())); + entityNode.set(key, JacksonUtil.toJsonNode(kvEntry.getJsonValue().get())); } } else { entityNode.put(key, kvEntry.getValueAsString()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index c516811e9c..2e525b56ed 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -171,7 +171,7 @@ public abstract class TbAbstractGetAttributesNode Date: Mon, 17 Oct 2022 12:36:13 +0300 Subject: [PATCH 11/26] add description --- .../rule/engine/metadata/TbGetAttributesNode.java | 6 +++--- .../rule/engine/metadata/TbGetDeviceAttrNode.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNode.java index 6388f9c44e..472b84c803 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNode.java @@ -34,9 +34,9 @@ import org.thingsboard.server.common.msg.TbMsg; @RuleNode(type = ComponentType.ENRICHMENT, name = "originator attributes", configClazz = TbGetAttributesNodeConfiguration.class, - nodeDescription = "Add Message Originator Attributes or Latest Telemetry into Message Metadata/data", - nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message metadata or data " + - "with specific prefix: cs/shared/ss. Latest telemetry value added into metadata/data without prefix. " + + nodeDescription = "Add Message Originator Attributes or Latest Telemetry into Message Data or Metadata", + nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message data/metadata " + + "with specific prefix: cs/shared/ss. Latest telemetry value added into Message data/metadata without prefix. " + "To access those attributes in other nodes this template can be used " + "metadata.cs_temperature or metadata.shared_limit ", uiResources = {"static/rulenode/rulenode-core-config.js"}, diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java index 5a48c8f95e..f38b5eccc7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java @@ -31,9 +31,9 @@ import org.thingsboard.server.common.msg.TbMsg; @RuleNode(type = ComponentType.ENRICHMENT, name = "related device attributes", configClazz = TbGetDeviceAttrNodeConfiguration.class, - nodeDescription = "Add Originators Related Device Attributes and Latest Telemetry value into Message Metadata", - nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message metadata " + - "with specific prefix: cs/shared/ss. Latest telemetry value added into metadata without prefix. " + + nodeDescription = "Add Originators Related Device Attributes and Latest Telemetry value into Message Data or Metadata", + nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message data/metadata " + + "with specific prefix: cs/shared/ss. Latest telemetry value added into Message data/metadata without prefix. " + "To access those attributes in other nodes this template can be used " + "metadata.cs_temperature or metadata.shared_limit ", uiResources = {"static/rulenode/rulenode-core-config.js"}, From 4c277daadd8a3941d903d65ae095c0a73d3e7518 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Fri, 28 Oct 2022 11:05:28 +0300 Subject: [PATCH 12/26] add test for TbAbstractGetAttributesNode --- .../metadata/TbAbstractGetAttributesNode.java | 2 +- .../TbAbstractGetAttributesNodeTest.java | 296 ++++++++++++++++++ 2 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNodeTest.java diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 2e525b56ed..e364ea2c18 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -93,7 +93,7 @@ public abstract class TbAbstractGetAttributesNode clientAttributes; + private List serverAttributes; + private List sharedAttributes; + private List tsKeys; + + @Before + public void before() throws TbNodeException { + dbExecutor = new AbstractListeningExecutor() { + @Override + protected int getThreadPollSize() { + return 3; + } + }; + dbExecutor.init(); + + Mockito.reset(ctx); + Mockito.reset(attributesService); + Mockito.reset(tsService); + + Mockito.reset(ctx); + Mockito.reset(attributesService); + Mockito.reset(tsService); + + lenient().when(ctx.getAttributesService()).thenReturn(attributesService); + lenient().when(ctx.getTimeseriesService()).thenReturn(tsService); + lenient().when(ctx.getTenantId()).thenReturn(tenantId); + lenient().when(ctx.getDbCallbackExecutor()).thenReturn(dbExecutor); + + clientAttributes = getAttributeNames("client"); + serverAttributes = getAttributeNames("server"); + sharedAttributes = getAttributeNames("shared"); + tsKeys = List.of("temperature", "humidity", "unknown"); + + Mockito.when(attributesService.find(tenantId, originator, DataConstants.CLIENT_SCOPE, clientAttributes)) + .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(clientAttributes))); + + + Mockito.when(attributesService.find(tenantId, originator, DataConstants.SERVER_SCOPE, serverAttributes)) + .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(serverAttributes))); + + + Mockito.when(attributesService.find(tenantId, originator, DataConstants.SHARED_SCOPE, sharedAttributes)) + .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(sharedAttributes))); + + Mockito.when(tsService.findLatest(tenantId, originator, tsKeys)) + .thenReturn(Futures.immediateFuture(getListTelemetryKvEntry(tsKeys))); + } + + @After + public void after() { + dbExecutor.destroy(); + } + + @Test + public void fetchToMetadata_whenOnMsg_then_success() throws Exception { + TbGetAttributesNode node = initNode(false, false, false); + TbMsg msg = getTbMsg(originator); + node.onMsg(ctx, msg); + + TbMsg resultMsg = checkMsg(); + TbMsgMetaData msgMetaData = resultMsg.getMetaData(); + + //check attributes + checkAttributes(clientAttributes, "cs_", false, msgMetaData, null); + checkAttributes(serverAttributes, "ss_", false, msgMetaData, null); + checkAttributes(sharedAttributes, "shared_", false, msgMetaData, null); + + //check timeseries + checkTs(tsKeys, false, false, msgMetaData, null); + } + + @Test + public void fetchToData_whenOnMsg_then_success() throws Exception { + TbGetAttributesNode node = initNode(true, true, false); + TbMsg msg = getTbMsg(originator); + node.onMsg(ctx, msg); + + TbMsg resultMsg = checkMsg(); + JsonNode msgData = JacksonUtil.toJsonNode(resultMsg.getData()); + + //check attributes + checkAttributes(clientAttributes, "cs_", true, null, msgData); + checkAttributes(serverAttributes, "ss_", true, null, msgData); + checkAttributes(sharedAttributes, "shared_", true, null, msgData); + + //check timeseries with ts + checkTs(tsKeys, true, true, null, msgData); + } + + @Test + public void fetchToData_whenOnMsg_then_failure() throws Exception { + TbGetAttributesNode node = initNode(true, true, true); + TbMsg msg = getTbMsg(originator); + node.onMsg(ctx, msg); + + ArgumentCaptor newMsgCaptor = ArgumentCaptor.forClass(TbMsg.class); + ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Exception.class); + Mockito.verify(ctx, never()).tellSuccess(any()); + Mockito.verify(ctx, Mockito.timeout(5000)).tellFailure(newMsgCaptor.capture(), exceptionCaptor.capture()); + + Assert.assertSame(newMsgCaptor.getValue(), msg); + Assert.assertNotNull(exceptionCaptor.getValue()); + } + + @Test + public void fetchToData_whenOnMsg_then_data_not_object_failure() throws Exception { + TbGetAttributesNode node = initNode(true, true, true); + TbMsg msg = TbMsg.newMsg("TEST", originator, new TbMsgMetaData(), "[]"); + node.onMsg(ctx, msg); + + ArgumentCaptor newMsgCaptor = ArgumentCaptor.forClass(TbMsg.class); + ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Exception.class); + Mockito.verify(ctx, never()).tellSuccess(any()); + Mockito.verify(ctx, Mockito.timeout(5000)).tellFailure(newMsgCaptor.capture(), exceptionCaptor.capture()); + + Assert.assertSame(newMsgCaptor.getValue(), msg); + Assert.assertNotNull(exceptionCaptor.getValue()); + } + + private TbMsg checkMsg() { + ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); + Mockito.verify(ctx, Mockito.timeout(5000)).tellSuccess(msgCaptor.capture()); + + TbMsg resultMsg = msgCaptor.getValue(); + Assert.assertNotNull(resultMsg); + Assert.assertNotNull(resultMsg.getMetaData()); + Assert.assertNotNull(resultMsg.getData()); + return resultMsg; + } + + private void checkAttributes(List attributes, String prefix, boolean fetchToData, TbMsgMetaData msgMetaData, JsonNode msgData) { + attributes.stream() + .filter(attribute -> !attribute.equals("unknown")) + .forEach(attribute -> { + String result; + if (fetchToData) { + result = msgData.get(prefix + attribute).asText(); + } else { + result = msgMetaData.getValue(prefix + attribute); + } + Assert.assertNotNull(result); + Assert.assertEquals(attribute + "_value", result); + }); + } + + private void checkTs(List tsKeys, boolean fetchToData, boolean getLatestValueWithTs, TbMsgMetaData msgMetaData, JsonNode msgData) { + long tsValue = 1L; + for (String key : tsKeys) { + if (key.equals("unknown")) { + continue; + } + String result; + if (fetchToData) { + if (getLatestValueWithTs) { + JsonNode resultTs = msgData.get(key); + Assert.assertNotNull(resultTs); + Assert.assertNotNull(resultTs.get("value")); + Assert.assertNotNull(resultTs.get("ts")); + result = resultTs.get("value").asText(); + } else { + result = msgData.get(key).asText(); + } + } else { + result = msgMetaData.getValue(key); + } + Assert.assertNotNull(result); + Assert.assertEquals(String.valueOf(tsValue), result); + tsValue++; + } + } + + private TbGetAttributesNode initNode(boolean fetchToData, boolean getLatestValueWithTs, boolean isTellFailureIfAbsent) throws TbNodeException { + TbGetAttributesNodeConfiguration config = new TbGetAttributesNodeConfiguration(); + config.setClientAttributeNames(List.of("client_attr_1", "client_attr_2", "${client_attr_metadata}", "unknown")); + config.setServerAttributeNames(List.of("server_attr_1", "server_attr_2", "${server_attr_metadata}", "unknown")); + config.setSharedAttributeNames(List.of("shared_attr_1", "shared_attr_2", "$[shared_attr_data]", "unknown")); + config.setLatestTsKeyNames(List.of("temperature", "humidity", "unknown")); + config.setFetchToData(fetchToData); + config.setGetLatestValueWithTs(getLatestValueWithTs); + config.setTellFailureIfAbsent(isTellFailureIfAbsent); + TbNodeConfiguration nodeConfiguration = new TbNodeConfiguration(mapper.valueToTree(config)); + TbGetAttributesNode node = new TbGetAttributesNode(); + node.init(ctx, nodeConfiguration); + return node; + } + + private TbMsg getTbMsg(EntityId entityId) { + ObjectNode msgData = JacksonUtil.newObjectNode(); + msgData.put("shared_attr_data", "shared_attr_3"); + + TbMsgMetaData msgMetaData = new TbMsgMetaData(); + msgMetaData.putValue("client_attr_metadata", "client_attr_3"); + msgMetaData.putValue("server_attr_metadata", "server_attr_3"); + + return TbMsg.newMsg("TEST", entityId, msgMetaData, msgData.toString()); + } + + private List getAttributeNames(String prefix) { + return List.of(prefix + "_attr_1", prefix + "_attr_2", prefix + "_attr_3", "unknown"); + } + + private List getListAttributeKvEntry(List attributes) { + List attributeKvEntries = new ArrayList<>(); + attributes.stream().filter(attribute -> !attribute.equals("unknown")).forEach(attribute -> attributeKvEntries.add(new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(attribute, attribute + "_value")))); + return attributeKvEntries; + } + + private List getListTelemetryKvEntry(List keys) { + long value = 1L; + List kvEntries = new ArrayList<>(); + for (String key : keys) { + if (key.equals("unknown")) { + continue; + } + kvEntries.add(new BasicTsKvEntry(System.currentTimeMillis(), new LongDataEntry(key, value))); + value++; + } + return kvEntries; + } + +} \ No newline at end of file From 36651d231da2db33ede7bd4e1a624c8c7e9aa70b Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Fri, 28 Oct 2022 11:06:00 +0300 Subject: [PATCH 13/26] mvn license:format --- .../rule/engine/metadata/TbAbstractGetAttributesNodeTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNodeTest.java index fc8f4ea5c9..e2914c8e60 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNodeTest.java @@ -55,7 +55,6 @@ import java.util.List; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; @RunWith(MockitoJUnitRunner.class) public class TbAbstractGetAttributesNodeTest { From 513d0f307587be117e8bb60da7eddc16c80f3c9f Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Mon, 31 Oct 2022 14:10:23 +0200 Subject: [PATCH 14/26] refactor code review --- .../rule/engine/metadata/TbAbstractGetAttributesNode.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index e364ea2c18..75f8c7732c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; import java.util.ArrayList; import java.util.HashMap; @@ -108,6 +109,7 @@ public abstract class TbAbstractGetAttributesNode { kvEntriesMap.forEach((keyScope, kvEntryList) -> { String prefix = getPrefix(keyScope); @@ -115,15 +117,15 @@ public abstract class TbAbstractGetAttributesNode ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } From 691339c5c5c907bbe9dbba8d0cce898dec991245 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Mon, 31 Oct 2022 16:03:27 +0200 Subject: [PATCH 15/26] update JacksonUtil method --- .../src/main/java/org/thingsboard/common/util/JacksonUtil.java | 2 +- .../rule/engine/metadata/TbAbstractGetAttributesNodeTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index 55d3c254d7..9fc83374a2 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -224,7 +224,7 @@ public class JacksonUtil { kvEntry.getLongValue().ifPresent(value -> entityNode.put(key, value)); } else if (kvEntry.getDataType() == DataType.JSON) { if (kvEntry.getJsonValue().isPresent()) { - entityNode.set(key, JacksonUtil.toJsonNode(kvEntry.getJsonValue().get())); + entityNode.set(key, toJsonNode(kvEntry.getJsonValue().get())); } } else { entityNode.put(key, kvEntry.getValueAsString()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNodeTest.java index e2914c8e60..a066c79343 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNodeTest.java @@ -292,4 +292,4 @@ public class TbAbstractGetAttributesNodeTest { return kvEntries; } -} \ No newline at end of file +} From 1d8733e55ad8b46345e4c5a4c6cb2cc244d7edb8 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Tue, 1 Nov 2022 16:54:25 +0200 Subject: [PATCH 16/26] return mapper --- .../metadata/TbAbstractGetAttributesNode.java | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 75f8c7732c..f50fe58dd5 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -15,7 +15,10 @@ */ package org.thingsboard.rule.engine.metadata; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.json.JsonWriteFeature; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -31,12 +34,14 @@ import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -54,6 +59,8 @@ import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; public abstract class TbAbstractGetAttributesNode implements TbNode { + private static ObjectMapper mapper = new ObjectMapper(); + private static final String VALUE = "value"; private static final String TS = "ts"; @@ -65,6 +72,8 @@ public abstract class TbAbstractGetAttributesNode { if (fetchToData) { - JacksonUtil.addKvEntry((ObjectNode) msgDataNode, kvEntry, prefix + kvEntry.getKey()); + addKvEntryToJson((ObjectNode) msgDataNode, kvEntry, prefix + kvEntry.getKey()); } else { msgMetaData.putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); } @@ -170,10 +179,10 @@ public abstract class TbAbstractGetAttributesNode entityNode.put(key, value)); + } else if (kvEntry.getDataType() == DataType.DOUBLE) { + kvEntry.getDoubleValue().ifPresent(value -> entityNode.put(key, value)); + } else if (kvEntry.getDataType() == DataType.LONG) { + kvEntry.getLongValue().ifPresent(value -> entityNode.put(key, value)); + } else if (kvEntry.getDataType() == DataType.JSON) { + if (kvEntry.getJsonValue().isPresent()) { + entityNode.set(key, toJsonNode(kvEntry.getJsonValue().get())); + } + } else { + entityNode.put(key, kvEntry.getValueAsString()); + } + } + + public static JsonNode toJsonNode(String value) { + try { + return mapper.readTree(value); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + private List getNotExistingKeys(List existingAttributesKvEntry, List allKeys) { List existingKeys = existingAttributesKvEntry.stream().map(KvEntry::getKey).collect(Collectors.toList()); return allKeys.stream().filter(key -> !existingKeys.contains(key)).collect(Collectors.toList()); From b74e5dfc4572da6de50579157cc8b4cf88de1fec Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Tue, 1 Nov 2022 16:56:08 +0200 Subject: [PATCH 17/26] change public to prv --- .../rule/engine/metadata/TbAbstractGetAttributesNode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index f50fe58dd5..c0564a98fe 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -201,7 +201,7 @@ public abstract class TbAbstractGetAttributesNode entityNode.put(key, value)); } else if (kvEntry.getDataType() == DataType.DOUBLE) { @@ -217,7 +217,7 @@ public abstract class TbAbstractGetAttributesNode Date: Tue, 1 Nov 2022 17:03:01 +0200 Subject: [PATCH 18/26] add JsonParseException --- .../rule/engine/metadata/TbAbstractGetAttributesNode.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index c0564a98fe..42a0666614 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.JsonParseException; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.thingsboard.common.util.JacksonUtil; @@ -221,7 +222,7 @@ public abstract class TbAbstractGetAttributesNode Date: Wed, 2 Nov 2022 11:44:46 +0200 Subject: [PATCH 19/26] added ALLOW_UNQUOTED_FIELD_NAMES_MAPPER to JacksonUtil --- .../thingsboard/common/util/JacksonUtil.java | 32 +++++++++++--- .../metadata/TbAbstractGetAttributesNode.java | 42 ++---------------- .../engine/metadata/TbGetTelemetryNode.java | 43 +++---------------- 3 files changed, 36 insertions(+), 81 deletions(-) diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index 9fc83374a2..dca796ee1a 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,7 +15,9 @@ */ package org.thingsboard.common.util; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.json.JsonWriteFeature; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.MapperFeature; @@ -46,6 +48,10 @@ public class JacksonUtil { .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true) .build(); + public static ObjectMapper ALLOW_UNQUOTED_FIELD_NAMES_MAPPER = JsonMapper.builder() + .configure(JsonWriteFeature.QUOTE_FIELD_NAMES.mappedFeature(), false) + .configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true) + .build(); public static T convertValue(Object fromValue, Class toValueType) { try { @@ -119,11 +125,15 @@ public class JacksonUtil { } public static JsonNode toJsonNode(String value) { + return toJsonNode(value, OBJECT_MAPPER); + } + + public static JsonNode toJsonNode(String value, ObjectMapper mapper) { if (value == null || value.isEmpty()) { return null; } try { - return OBJECT_MAPPER.readTree(value); + return mapper.readTree(value); } catch (IOException e) { throw new IllegalArgumentException(e); } @@ -138,7 +148,11 @@ public class JacksonUtil { } public static ObjectNode newObjectNode() { - return OBJECT_MAPPER.createObjectNode(); + return newObjectNode(OBJECT_MAPPER); + } + + public static ObjectNode newObjectNode(ObjectMapper mapper) { + return mapper.createObjectNode(); } public static T clone(T value) { @@ -216,6 +230,10 @@ public class JacksonUtil { } public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry, String key) { + addKvEntry(entityNode, kvEntry, key, OBJECT_MAPPER); + } + + public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry, String key, ObjectMapper mapper) { if (kvEntry.getDataType() == DataType.BOOLEAN) { kvEntry.getBooleanValue().ifPresent(value -> entityNode.put(key, value)); } else if (kvEntry.getDataType() == DataType.DOUBLE) { @@ -224,7 +242,7 @@ public class JacksonUtil { kvEntry.getLongValue().ifPresent(value -> entityNode.put(key, value)); } else if (kvEntry.getDataType() == DataType.JSON) { if (kvEntry.getJsonValue().isPresent()) { - entityNode.set(key, toJsonNode(kvEntry.getJsonValue().get())); + entityNode.set(key, toJsonNode(kvEntry.getJsonValue().get(), mapper)); } } else { entityNode.put(key, kvEntry.getValueAsString()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 42a0666614..1b94bbfd2b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -15,15 +15,11 @@ */ package org.thingsboard.rule.engine.metadata; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.json.JsonWriteFeature; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.JsonParseException; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.thingsboard.common.util.JacksonUtil; @@ -35,14 +31,12 @@ import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -60,8 +54,6 @@ import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; public abstract class TbAbstractGetAttributesNode implements TbNode { - private static ObjectMapper mapper = new ObjectMapper(); - private static final String VALUE = "value"; private static final String TS = "ts"; @@ -73,8 +65,6 @@ public abstract class TbAbstractGetAttributesNode { if (fetchToData) { - addKvEntryToJson((ObjectNode) msgDataNode, kvEntry, prefix + kvEntry.getKey()); + JacksonUtil.addKvEntry((ObjectNode) msgDataNode, kvEntry, prefix + kvEntry.getKey(), JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER); } else { msgMetaData.putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); } @@ -180,9 +170,9 @@ public abstract class TbAbstractGetAttributesNode entityNode.put(key, value)); - } else if (kvEntry.getDataType() == DataType.DOUBLE) { - kvEntry.getDoubleValue().ifPresent(value -> entityNode.put(key, value)); - } else if (kvEntry.getDataType() == DataType.LONG) { - kvEntry.getLongValue().ifPresent(value -> entityNode.put(key, value)); - } else if (kvEntry.getDataType() == DataType.JSON) { - if (kvEntry.getJsonValue().isPresent()) { - entityNode.set(key, toJsonNode(kvEntry.getJsonValue().get())); - } - } else { - entityNode.put(key, kvEntry.getValueAsString()); - } - } - - private static JsonNode toJsonNode(String value) { - try { - return mapper.readTree(value); - } catch (IOException e) { - throw new JsonParseException("Can't parse jsonValue: " + value, e); - } - } - private List getNotExistingKeys(List existingAttributesKvEntry, List allKeys) { List existingKeys = existingAttributesKvEntry.stream().map(KvEntry::getKey).collect(Collectors.toList()); return allKeys.stream().filter(key -> !existingKeys.contains(key)).collect(Collectors.toList()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index 77298cba77..045555eae8 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -15,25 +15,22 @@ */ package org.thingsboard.rule.engine.metadata; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.json.JsonWriteFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.ListenableFuture; -import com.google.gson.JsonParseException; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.thingsboard.common.util.DonAsynchron; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; @@ -41,7 +38,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import java.io.IOException; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -75,7 +71,6 @@ public class TbGetTelemetryNode implements TbNode { private TbGetTelemetryNodeConfiguration config; private List tsKeyNames; private int limit; - private ObjectMapper mapper; private String fetchMode; private String orderByFetchAll; private Aggregation aggregation; @@ -91,10 +86,6 @@ public class TbGetTelemetryNode implements TbNode { orderByFetchAll = ASC_ORDER; } aggregation = parseAggregationConfig(config.getAggregation()); - - mapper = new ObjectMapper(); - mapper.configure(JsonWriteFeature.QUOTE_FIELD_NAMES.mappedFeature(), false); - mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); } Aggregation parseAggregationConfig(String aggName) { @@ -146,7 +137,7 @@ public class TbGetTelemetryNode implements TbNode { } private void process(List entries, TbMsg msg, List keys) { - ObjectNode resultNode = mapper.createObjectNode(); + ObjectNode resultNode = JacksonUtil.newObjectNode(JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER); if (FETCH_MODE_ALL.equals(fetchMode)) { entries.forEach(entry -> processArray(resultNode, entry)); } else { @@ -169,36 +160,16 @@ public class TbGetTelemetryNode implements TbNode { ArrayNode arrayNode = (ArrayNode) node.get(entry.getKey()); arrayNode.add(buildNode(entry)); } else { - ArrayNode arrayNode = mapper.createArrayNode(); + ArrayNode arrayNode = JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER.createArrayNode(); arrayNode.add(buildNode(entry)); node.set(entry.getKey(), arrayNode); } } private ObjectNode buildNode(TsKvEntry entry) { - ObjectNode obj = mapper.createObjectNode() - .put("ts", entry.getTs()); - switch (entry.getDataType()) { - case STRING: - obj.put("value", entry.getValueAsString()); - break; - case LONG: - obj.put("value", entry.getLongValue().get()); - break; - case BOOLEAN: - obj.put("value", entry.getBooleanValue().get()); - break; - case DOUBLE: - obj.put("value", entry.getDoubleValue().get()); - break; - case JSON: - try { - obj.set("value", mapper.readTree(entry.getJsonValue().get())); - } catch (IOException e) { - throw new JsonParseException("Can't parse jsonValue: " + entry.getJsonValue().get(), e); - } - break; - } + ObjectNode obj = JacksonUtil.newObjectNode(JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER); + obj.put("ts", entry.getTs()); + JacksonUtil.addKvEntry(obj, entry, "value", JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER); return obj; } From 41a5f8d9e4be9a9c47f0508ee44f2be90c4530cd Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Wed, 2 Nov 2022 11:49:05 +0200 Subject: [PATCH 20/26] fix license for JacksonUtil --- .../java/org/thingsboard/common/util/JacksonUtil.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index dca796ee1a..1f21435bd3 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From fea91220f03074e61d7b7b4b1a31899baea6dd0c Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Wed, 2 Nov 2022 12:55:03 +0200 Subject: [PATCH 21/26] fix error in merge with master --- .../BaseDeviceProfileControllerTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java index 10b1ff0e74..3fe80b1112 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.device.profile.JsonTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleChain; @@ -983,4 +984,21 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(errorMsg)); } + @Test + public void testDeleteDeviceProfileWithDeleteRelationsOk() throws Exception { + DeviceProfileId deviceProfileId = savedDeviceProfile("DeviceProfile for Test WithRelationsOk").getId(); + testEntityDaoWithRelationsOk(savedTenant.getId(), deviceProfileId, "/api/deviceProfile/" + deviceProfileId); + } + + @Test + public void testDeleteDeviceProfileExceptionWithRelationsTransactional() throws Exception { + DeviceProfileId deviceProfileId = savedDeviceProfile("DeviceProfile for Test WithRelations Transactional Exception").getId(); + testEntityDaoWithRelationsTransactionalException(deviceProfileDao, savedTenant.getId(), deviceProfileId, "/api/deviceProfile/" + deviceProfileId); + } + + private DeviceProfile savedDeviceProfile(String name) { + DeviceProfile deviceProfile = createDeviceProfile(name); + return doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + } + } From e94d305f61c03737065012a31080bd0885cf9277 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Thu, 3 Nov 2022 12:31:31 +0200 Subject: [PATCH 22/26] added JacksonUtilTest for ALLOW_UNQUOTED_FIELD_NAMES_MAPPER and updated getAttributes tests --- .../thingsboard/common/util/JacksonUtil.java | 7 +- .../common/util/JacksonUtilTest.java | 35 +++++++ .../metadata/TbAbstractGetAttributesNode.java | 22 +++-- .../TbAbstractGetAttributesNodeTest.java | 95 ++++++++++++++----- 4 files changed, 122 insertions(+), 37 deletions(-) create mode 100644 common/util/src/test/java/org/thingsboard/common/util/JacksonUtilTest.java diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index 1f21435bd3..f80af0789e 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -229,6 +229,10 @@ public class JacksonUtil { } } + public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry) { + addKvEntry(entityNode, kvEntry, kvEntry.getKey()); + } + public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry, String key) { addKvEntry(entityNode, kvEntry, key, OBJECT_MAPPER); } @@ -249,7 +253,4 @@ public class JacksonUtil { } } - public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry) { - addKvEntry(entityNode, kvEntry, kvEntry.getKey()); - } } diff --git a/common/util/src/test/java/org/thingsboard/common/util/JacksonUtilTest.java b/common/util/src/test/java/org/thingsboard/common/util/JacksonUtilTest.java new file mode 100644 index 0000000000..22cd1b3ff6 --- /dev/null +++ b/common/util/src/test/java/org/thingsboard/common/util/JacksonUtilTest.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.common.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Assert; +import org.junit.Test; + +public class JacksonUtilTest { + + @Test + public void allow_unquoted_field_mapper_test() { + String data = "{data: 123}"; + JsonNode actualResult = JacksonUtil.toJsonNode(data, JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER); // should be: {"data": 123} + ObjectNode expectedResult = JacksonUtil.newObjectNode(); + expectedResult.put("data", 123); // {"data": 123} + Assert.assertEquals(expectedResult, actualResult); + Assert.assertThrows(IllegalArgumentException.class, () -> JacksonUtil.toJsonNode(data)); // syntax exception due to missing quotes in the field name! + } + +} \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 1b94bbfd2b..32f2d16484 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -16,6 +16,7 @@ package org.thingsboard.rule.engine.metadata; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -91,16 +92,19 @@ public abstract class TbAbstractGetAttributesNode> failuresMap = new ConcurrentHashMap<>(); ListenableFuture>>> allFutures = Futures.allAsList( - getLatestTelemetry(ctx, entityId, msg, LATEST_TS, TbNodeUtils.processPatterns(config.getLatestTsKeyNames(), msg), failuresMap), + getLatestTelemetry(ctx, entityId, TbNodeUtils.processPatterns(config.getLatestTsKeyNames(), msg), failuresMap), getAttrAsync(ctx, entityId, CLIENT_SCOPE, TbNodeUtils.processPatterns(config.getClientAttributeNames(), msg), failuresMap), getAttrAsync(ctx, entityId, SHARED_SCOPE, TbNodeUtils.processPatterns(config.getSharedAttributeNames(), msg), failuresMap), getAttrAsync(ctx, entityId, SERVER_SCOPE, TbNodeUtils.processPatterns(config.getServerAttributeNames(), msg), failuresMap) @@ -114,10 +118,11 @@ public abstract class TbAbstractGetAttributesNode { String prefix = getPrefix(keyScope); kvEntryList.forEach(kvEntry -> { + String key = prefix + kvEntry.getKey(); if (fetchToData) { - JacksonUtil.addKvEntry((ObjectNode) msgDataNode, kvEntry, prefix + kvEntry.getKey(), JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER); + JacksonUtil.addKvEntry((ObjectNode) msgDataNode, kvEntry, key); } else { - msgMetaData.putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString()); + msgMetaData.putValue(key, kvEntry.getValueAsString()); } }); }); @@ -145,7 +150,7 @@ public abstract class TbAbstractGetAttributesNode>> getLatestTelemetry(TbContext ctx, EntityId entityId, TbMsg msg, String scope, List keys, ConcurrentHashMap> failuresMap) { + private ListenableFuture>> getLatestTelemetry(TbContext ctx, EntityId entityId, List keys, ConcurrentHashMap> failuresMap) { if (CollectionUtils.isEmpty(keys)) { return Futures.immediateFuture(null); } @@ -155,7 +160,7 @@ public abstract class TbAbstractGetAttributesNode { if (tsKvEntry.getValue() == null) { if (isTellFailureIfAbsent) { - computeFailuresMap(scope, failuresMap, tsKvEntry.getKey()); + computeFailuresMap(LATEST_TS, failuresMap, tsKvEntry.getKey()); } } else if (getLatestValueWithTs) { listTsKvEntry.add(getValueWithTs(tsKvEntry)); @@ -164,7 +169,7 @@ public abstract class TbAbstractGetAttributesNode> mapTsKvEntry = new HashMap<>(); - mapTsKvEntry.put(scope, listTsKvEntry); + mapTsKvEntry.put(LATEST_TS, listTsKvEntry); return mapTsKvEntry; }, MoreExecutors.directExecutor()); } @@ -172,7 +177,8 @@ public abstract class TbAbstractGetAttributesNode serverAttributes; private List sharedAttributes; private List tsKeys; + private long ts; @Before public void before() throws TbNodeException { @@ -104,20 +106,21 @@ public class TbAbstractGetAttributesNodeTest { serverAttributes = getAttributeNames("server"); sharedAttributes = getAttributeNames("shared"); tsKeys = List.of("temperature", "humidity", "unknown"); + ts = System.currentTimeMillis(); Mockito.when(attributesService.find(tenantId, originator, DataConstants.CLIENT_SCOPE, clientAttributes)) - .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(clientAttributes))); + .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(clientAttributes, ts))); Mockito.when(attributesService.find(tenantId, originator, DataConstants.SERVER_SCOPE, serverAttributes)) - .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(serverAttributes))); + .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(serverAttributes, ts))); Mockito.when(attributesService.find(tenantId, originator, DataConstants.SHARED_SCOPE, sharedAttributes)) - .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(sharedAttributes))); + .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(sharedAttributes, ts))); Mockito.when(tsService.findLatest(tenantId, originator, tsKeys)) - .thenReturn(Futures.immediateFuture(getListTelemetryKvEntry(tsKeys))); + .thenReturn(Futures.immediateFuture(getListTsKvEntry(tsKeys, ts))); } @After @@ -143,8 +146,44 @@ public class TbAbstractGetAttributesNodeTest { checkTs(tsKeys, false, false, msgMetaData, null); } + @Test + public void fetchToMetadata_latestWithTs_whenOnMsg_then_success() throws Exception { + TbGetAttributesNode node = initNode(false, true, false); + TbMsg msg = getTbMsg(originator); + node.onMsg(ctx, msg); + + TbMsg resultMsg = checkMsg(); + TbMsgMetaData msgMetaData = resultMsg.getMetaData(); + + //check attributes + checkAttributes(clientAttributes, "cs_", false, msgMetaData, null); + checkAttributes(serverAttributes, "ss_", false, msgMetaData, null); + checkAttributes(sharedAttributes, "shared_", false, msgMetaData, null); + + //check timeseries with ts + checkTs(tsKeys, false, true, msgMetaData, null); + } + @Test public void fetchToData_whenOnMsg_then_success() throws Exception { + TbGetAttributesNode node = initNode(true, false, false); + TbMsg msg = getTbMsg(originator); + node.onMsg(ctx, msg); + + TbMsg resultMsg = checkMsg(); + JsonNode msgData = JacksonUtil.toJsonNode(resultMsg.getData()); + + //check attributes + checkAttributes(clientAttributes, "cs_", true, null, msgData); + checkAttributes(serverAttributes, "ss_", true, null, msgData); + checkAttributes(sharedAttributes, "shared_", true, null, msgData); + + //check timeseries + checkTs(tsKeys, true, false, null, msgData); + } + + @Test + public void fetchToData_latestWithTs_whenOnMsg_then_success() throws Exception { TbGetAttributesNode node = initNode(true, true, false); TbMsg msg = getTbMsg(originator); node.onMsg(ctx, msg); @@ -218,28 +257,26 @@ public class TbAbstractGetAttributesNodeTest { } private void checkTs(List tsKeys, boolean fetchToData, boolean getLatestValueWithTs, TbMsgMetaData msgMetaData, JsonNode msgData) { - long tsValue = 1L; + long value = 1L; for (String key : tsKeys) { if (key.equals("unknown")) { continue; } - String result; + String actualValue; + String expectedValue; + if (getLatestValueWithTs) { + expectedValue = "{\"ts\":" + ts + ",\"value\":{\"data\":" + value + "}}"; + } else { + expectedValue = "{\"data\":" + value + "}"; + } if (fetchToData) { - if (getLatestValueWithTs) { - JsonNode resultTs = msgData.get(key); - Assert.assertNotNull(resultTs); - Assert.assertNotNull(resultTs.get("value")); - Assert.assertNotNull(resultTs.get("ts")); - result = resultTs.get("value").asText(); - } else { - result = msgData.get(key).asText(); - } + actualValue = JacksonUtil.toString(msgData.get(key)); } else { - result = msgMetaData.getValue(key); + actualValue = msgMetaData.getValue(key); } - Assert.assertNotNull(result); - Assert.assertEquals(String.valueOf(tsValue), result); - tsValue++; + Assert.assertNotNull(actualValue); + Assert.assertEquals(expectedValue, actualValue); + value++; } } @@ -273,20 +310,26 @@ public class TbAbstractGetAttributesNodeTest { return List.of(prefix + "_attr_1", prefix + "_attr_2", prefix + "_attr_3", "unknown"); } - private List getListAttributeKvEntry(List attributes) { - List attributeKvEntries = new ArrayList<>(); - attributes.stream().filter(attribute -> !attribute.equals("unknown")).forEach(attribute -> attributeKvEntries.add(new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(attribute, attribute + "_value")))); - return attributeKvEntries; + private List getListAttributeKvEntry(List attributes, long ts) { + return attributes.stream() + .filter(attribute -> !attribute.equals("unknown")) + .map(attribute -> toAttributeKvEntry(ts, attribute)) + .collect(Collectors.toList()); + } + + private BaseAttributeKvEntry toAttributeKvEntry(long ts, String attribute) { + return new BaseAttributeKvEntry(ts, new StringDataEntry(attribute, attribute + "_value")); } - private List getListTelemetryKvEntry(List keys) { + private List getListTsKvEntry(List keys, long ts) { long value = 1L; List kvEntries = new ArrayList<>(); for (String key : keys) { if (key.equals("unknown")) { continue; } - kvEntries.add(new BasicTsKvEntry(System.currentTimeMillis(), new LongDataEntry(key, value))); + String dataValue = "{\"data\":" + value + "}"; + kvEntries.add(new BasicTsKvEntry(ts, new JsonDataEntry(key, dataValue))); value++; } return kvEntries; From a5691f9347b8b54cb8d7df461de3b96d10506fcd Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 3 Nov 2022 15:58:22 +0200 Subject: [PATCH 23/26] Fix for the attributes mapper --- .../rule/engine/metadata/TbAbstractGetAttributesNode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 32f2d16484..d9352f6e4e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -175,9 +175,9 @@ public abstract class TbAbstractGetAttributesNode Date: Thu, 3 Nov 2022 17:47:54 +0200 Subject: [PATCH 24/26] UI: Refactoring code --- ui-ngx/src/app/core/services/utils.service.ts | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/ui-ngx/src/app/core/services/utils.service.ts b/ui-ngx/src/app/core/services/utils.service.ts index c02abcdd39..66eefa9e13 100644 --- a/ui-ngx/src/app/core/services/utils.service.ts +++ b/ui-ngx/src/app/core/services/utils.service.ts @@ -47,9 +47,9 @@ import { publishReplay, refCount } from 'rxjs/operators'; import { WidgetContext } from '@app/modules/home/models/widget-component.models'; import { AttributeData, + LatestTelemetry, TelemetrySubscriber, - TelemetryType, - LatestTelemetry + TelemetryType } from '@shared/models/telemetry/telemetry.models'; import { EntityId } from '@shared/models/id/entity-id'; @@ -489,25 +489,18 @@ export class UtilsService { } } - public getEntityIdFromDatasource(dataSource: Datasource): EntityId { + private getEntityIdFromDatasource(dataSource: Datasource): EntityId { return {id: dataSource.entityId, entityType: dataSource.entityType}; } - public createTelemetrySubscriber(ctx: WidgetContext, - entityId?: EntityId, - type: TelemetryType = LatestTelemetry.LATEST_TELEMETRY, - keys: string[] = null): TelemetrySubscriber { - if (!entityId && ctx.datasources.length > 0) { - entityId = this.getEntityIdFromDatasource(ctx.datasources[0]); - } - return TelemetrySubscriber.createEntityAttributesSubscription(ctx.telemetryWsService, entityId, type, ctx.ngZone, keys); - } - public subscribeToEntityTelemetry(ctx: WidgetContext, entityId?: EntityId, type: TelemetryType = LatestTelemetry.LATEST_TELEMETRY, keys: string[] = null): Observable> { - const subscription = this.createTelemetrySubscriber(ctx, entityId, type, keys); + if (!entityId && ctx.datasources.length > 0) { + entityId = this.getEntityIdFromDatasource(ctx.datasources[0]); + } + const subscription = TelemetrySubscriber.createEntityAttributesSubscription(ctx.telemetryWsService, entityId, type, ctx.ngZone, keys); if (!ctx.telemetrySubscribers) { ctx.telemetrySubscribers = []; } From 7ca50ac3e5c46e55b9d00e5aabea0ab37816b8d6 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 3 Nov 2022 19:37:56 +0200 Subject: [PATCH 25/26] MVEL engine warm up improvement --- .../script/api/mvel/DefaultMvelInvokeService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java index e72c24ecf4..393f935861 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java @@ -24,7 +24,6 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.mvel2.ExecutionContext; import org.mvel2.MVEL; -import org.mvel2.ParserContext; import org.mvel2.SandboxedParserConfiguration; import org.mvel2.SandboxedParserContext; import org.mvel2.ScriptMemoryOverflowException; @@ -44,6 +43,7 @@ import org.thingsboard.server.common.stats.TbApiUsageStateClient; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.Serializable; +import java.util.Collections; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -115,7 +115,8 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "mvel-executor")); try { // Special command to warm up MVEL engine - MVEL.compileExpression("var warmUp = {}; warmUp", new SandboxedParserContext(parserConfig)); + Serializable script = MVEL.compileExpression("var warmUp = {}; warmUp", new SandboxedParserContext(parserConfig)); + MVEL.executeTbExpression(script, new ExecutionContext(), Collections.emptyMap()); } catch (Exception e) { // do nothing } From e705b5a2a49ef05781d32452395db4107c4897f7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 4 Nov 2022 13:10:15 +0200 Subject: [PATCH 26/26] TbMsgProcessingStackItem implements Serializable --- .../common/msg/TbMsgProcessingStackItem.java | 3 +- .../msg/TbMsgProcessingStackItemTest.java | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 common/message/src/test/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItemTest.java diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItem.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItem.java index 7c7f300778..f17428f0bf 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItem.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItem.java @@ -20,10 +20,11 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; +import java.io.Serializable; import java.util.UUID; @Data -public class TbMsgProcessingStackItem { +public class TbMsgProcessingStackItem implements Serializable { private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; diff --git a/common/message/src/test/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItemTest.java b/common/message/src/test/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItemTest.java new file mode 100644 index 0000000000..8f8e59cd69 --- /dev/null +++ b/common/message/src/test/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItemTest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg; + +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.FSTUtils; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +class TbMsgProcessingStackItemTest { + + @Test + void testSerialization() { + TbMsgProcessingStackItem item = new TbMsgProcessingStackItem(new RuleChainId(UUID.randomUUID()), new RuleNodeId(UUID.randomUUID())); + byte[] bytes = FSTUtils.encode(item); + TbMsgProcessingStackItem itemDecoded = FSTUtils.decode(bytes); + assertThat(item).isEqualTo(itemDecoded); + } + +}