diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java index b811497b9f..3fe80b1112 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java @@ -16,12 +16,6 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -46,11 +40,9 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.JsonTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; -import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -62,13 +54,9 @@ import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; import static org.thingsboard.server.common.data.ota.OtaPackageType.SOFTWARE; @@ -749,137 +737,6 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController "}", "[Transport Configuration] invalid attributes proto schema provided! OneOf definition groups don't support!"); } - @Test - public void testSaveProtoDeviceProfileWithMessageNestedTypes() throws Exception { - String schema = "syntax = \"proto3\";\n" + - "\n" + - "package testnested;\n" + - "\n" + - "message Outer {\n" + - " message MiddleAA {\n" + - " message Inner {\n" + - " optional int64 ival = 1;\n" + - " optional bool booly = 2;\n" + - " }\n" + - " Inner inner = 1;\n" + - " }\n" + - " message MiddleBB {\n" + - " message Inner {\n" + - " optional int32 ival = 1;\n" + - " optional bool booly = 2;\n" + - " }\n" + - " Inner inner = 1;\n" + - " }\n" + - " MiddleAA middleAA = 1;\n" + - " MiddleBB middleBB = 2;\n" + - "}"; - DynamicSchema dynamicSchema = getDynamicSchema(schema); - assertNotNull(dynamicSchema); - Set messageTypes = dynamicSchema.getMessageTypes(); - assertEquals(5, messageTypes.size()); - assertTrue(messageTypes.contains("testnested.Outer")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleAA")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleAA.Inner")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleBB")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleBB.Inner")); - - DynamicMessage.Builder middleAAInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); - Descriptors.Descriptor middleAAInnerMsgDescriptor = middleAAInnerMsgBuilder.getDescriptorForType(); - DynamicMessage middleAAInnerMsg = middleAAInnerMsgBuilder - .setField(middleAAInnerMsgDescriptor.findFieldByName("ival"), 1L) - .setField(middleAAInnerMsgDescriptor.findFieldByName("booly"), true) - .build(); - - DynamicMessage.Builder middleAAMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA"); - Descriptors.Descriptor middleAAMsgDescriptor = middleAAMsgBuilder.getDescriptorForType(); - DynamicMessage middleAAMsg = middleAAMsgBuilder - .setField(middleAAMsgDescriptor.findFieldByName("inner"), middleAAInnerMsg) - .build(); - - DynamicMessage.Builder middleBBInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); - Descriptors.Descriptor middleBBInnerMsgDescriptor = middleBBInnerMsgBuilder.getDescriptorForType(); - DynamicMessage middleBBInnerMsg = middleBBInnerMsgBuilder - .setField(middleBBInnerMsgDescriptor.findFieldByName("ival"), 0L) - .setField(middleBBInnerMsgDescriptor.findFieldByName("booly"), false) - .build(); - - DynamicMessage.Builder middleBBMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleBB"); - Descriptors.Descriptor middleBBMsgDescriptor = middleBBMsgBuilder.getDescriptorForType(); - DynamicMessage middleBBMsg = middleBBMsgBuilder - .setField(middleBBMsgDescriptor.findFieldByName("inner"), middleBBInnerMsg) - .build(); - - - DynamicMessage.Builder outerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer"); - Descriptors.Descriptor outerMsgBuilderDescriptor = outerMsgBuilder.getDescriptorForType(); - DynamicMessage outerMsg = outerMsgBuilder - .setField(outerMsgBuilderDescriptor.findFieldByName("middleAA"), middleAAMsg) - .setField(outerMsgBuilderDescriptor.findFieldByName("middleBB"), middleBBMsg) - .build(); - - assertEquals("{\n" + - " \"middleAA\": {\n" + - " \"inner\": {\n" + - " \"ival\": \"1\",\n" + - " \"booly\": true\n" + - " }\n" + - " },\n" + - " \"middleBB\": {\n" + - " \"inner\": {\n" + - " \"ival\": 0,\n" + - " \"booly\": false\n" + - " }\n" + - " }\n" + - "}", dynamicMsgToJson(outerMsgBuilderDescriptor, outerMsg.toByteArray())); - } - - @Test - public void testSaveProtoDeviceProfileWithMessageOneOfs() throws Exception { - String schema = "syntax = \"proto3\";\n" + - "\n" + - "package testoneofs;\n" + - "\n" + - "message SubMessage {\n" + - " repeated string name = 1;\n" + - "}\n" + - "\n" + - "message SampleMessage {\n" + - " optional int32 id = 1;\n" + - " oneof testOneOf {\n" + - " string name = 4;\n" + - " SubMessage subMessage = 9;\n" + - " }\n" + - "}"; - DynamicSchema dynamicSchema = getDynamicSchema(schema); - assertNotNull(dynamicSchema); - Set messageTypes = dynamicSchema.getMessageTypes(); - assertEquals(2, messageTypes.size()); - assertTrue(messageTypes.contains("testoneofs.SubMessage")); - assertTrue(messageTypes.contains("testoneofs.SampleMessage")); - - DynamicMessage.Builder sampleMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SampleMessage"); - Descriptors.Descriptor sampleMsgDescriptor = sampleMsgBuilder.getDescriptorForType(); - assertNotNull(sampleMsgDescriptor); - - List fields = sampleMsgDescriptor.getFields(); - assertEquals(3, fields.size()); - DynamicMessage sampleMsg = sampleMsgBuilder - .setField(sampleMsgDescriptor.findFieldByName("name"), "Bob") - .build(); - assertEquals("{\n" + " \"name\": \"Bob\"\n" + "}", dynamicMsgToJson(sampleMsgDescriptor, sampleMsg.toByteArray())); - - DynamicMessage.Builder subMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SubMessage"); - Descriptors.Descriptor subMsgDescriptor = subMsgBuilder.getDescriptorForType(); - DynamicMessage subMsg = subMsgBuilder - .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "Alice") - .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "John") - .build(); - - DynamicMessage sampleMsgWithOneOfSubMessage = sampleMsgBuilder.setField(sampleMsgDescriptor.findFieldByName("subMessage"), subMsg).build(); - assertEquals("{\n" + " \"subMessage\": {\n" + " \"name\": [\"Alice\", \"John\"]\n" + " }\n" + "}", - dynamicMsgToJson(sampleMsgDescriptor, sampleMsgWithOneOfSubMessage.toByteArray())); - } - @Test public void testSaveProtoDeviceProfileWithInvalidTelemetrySchemaTsField() throws Exception { testSaveDeviceProfileWithInvalidProtoSchema("syntax =\"proto3\";\n" + @@ -1127,23 +984,6 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(errorMsg)); } - private DynamicSchema getDynamicSchema(String schema) throws Exception { - DeviceProfile deviceProfile = testSaveDeviceProfileWithProtoPayloadType(schema); - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration); - MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttDeviceProfileTransportConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement protoFile = protoTransportPayloadConfiguration.getTransportProtoSchema(schema); - return protoTransportPayloadConfiguration.getDynamicSchema(protoFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - } - - private String dynamicMsgToJson(Descriptors.Descriptor descriptor, byte[] payload) throws InvalidProtocolBufferException { - DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, payload); - return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage); - } - @Test public void testDeleteDeviceProfileWithDeleteRelationsOk() throws Exception { DeviceProfileId deviceProfileId = savedDeviceProfile("DeviceProfile for Test WithRelationsOk").getId(); diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java index cd9ed239a5..7617f2b826 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java @@ -27,6 +27,7 @@ import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; @@ -124,7 +125,6 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap } private byte[] getAttributesProtoPayloadBytes() { - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; @@ -134,8 +134,8 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema()); + DynamicSchema attributesSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java index 2d034d3b41..41001427a5 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java @@ -28,6 +28,7 @@ import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -158,8 +159,8 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC protected void processOnLoadProtoResponse(CoapResponse response, CoapTestClient client, Integer observe, CountDownLatch latch) { ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); - ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); - DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); + ProtoFileElement rpcRequestProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcRequestProtoSchema()); + DynamicSchema rpcRequestProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcRequestProtoFileElement, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); byte[] requestPayload = response.getPayload(); DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); @@ -168,8 +169,8 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload); Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId"); int requestId = (int) dynamicMessage.getField(requestIdDescriptor); - ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); - DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); + ProtoFileElement rpcResponseProtoSchemaFile = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcResponseProtoSchema()); + DynamicSchema rpcResponseProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); DynamicMessage rpcResponseMsg = rpcResponseBuilder diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesProtoIntegrationTest.java index 539f62f614..609a5fe672 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesProtoIntegrationTest.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -55,18 +56,7 @@ public class CoapAttributesProtoIntegrationTest extends CoapAttributesIntegratio @Test public void testPushAttributes() throws Exception { - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - + DynamicSchema attributesSchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); assertNotNull(nestedJsonObjectBuilderDescriptor); @@ -98,18 +88,7 @@ public class CoapAttributesProtoIntegrationTest extends CoapAttributesIntegratio @Test public void testPushAttributesWithExplicitPresenceProtoKeys() throws Exception { - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - + DynamicSchema attributesSchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); assertNotNull(nestedJsonObjectBuilderDescriptor); @@ -135,4 +114,19 @@ public class CoapAttributesProtoIntegrationTest extends CoapAttributesIntegratio processAttributesTest(Arrays.asList("key1", "key5"), postAttributesMsg.toByteArray(), true); } + private DynamicSchema getDynamicSchema() { + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); + CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; + CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); + assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); + DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + String deviceAttributesProtoSchema = protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceAttributesProtoSchema); + return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + } + } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java index fb6d8aa0da..536155f1cf 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java @@ -23,6 +23,8 @@ import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -63,8 +65,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -138,8 +141,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(schemaStr); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -198,8 +202,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -271,8 +276,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(schemaStr); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java index 6d030aca9f..56d9ef5e9e 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; @@ -494,8 +495,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema()); + DynamicSchema attributesSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 067af192e4..2cdfb76645 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -29,6 +29,7 @@ import org.eclipse.paho.client.mqttv3.MqttException; import org.eclipse.paho.client.mqttv3.MqttMessage; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TransportPayloadType; @@ -382,8 +383,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractM protected byte[] processProtoMessageArrived(String requestTopic, MqttMessage mqttMessage) throws MqttException, InvalidProtocolBufferException { if (requestTopic.startsWith(BASE_DEVICE_API_TOPIC) || requestTopic.startsWith(BASE_DEVICE_API_TOPIC_V2)) { ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); - ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); - DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); + ProtoFileElement rpcRequestProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcRequestProtoSchema()); + DynamicSchema rpcRequestProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcRequestProtoFileElement, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); byte[] requestPayload = mqttMessage.getPayload(); DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); @@ -395,8 +396,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractM for (Descriptors.FieldDescriptor fieldDescriptor: fields) { assertTrue(dynamicMessage.hasField(fieldDescriptor)); } - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); - DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); + ProtoFileElement rpcResponseProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcResponseProtoSchema()); + DynamicSchema rpcResponseProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcResponseProtoFileElement, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/MqttAttributesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/MqttAttributesProtoIntegrationTest.java index 4904ed7a5d..7b2fa1143d 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/MqttAttributesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/MqttAttributesProtoIntegrationTest.java @@ -22,6 +22,7 @@ import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; @@ -176,8 +177,9 @@ public class MqttAttributesProtoIntegrationTest extends MqttAttributesIntegratio TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA); - return protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + String deviceAttributesProtoSchema = protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceAttributesProtoSchema); + return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); } private DynamicMessage getDefaultDynamicMessage() { diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java index 95de5f0069..9c92e3f4f0 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; @@ -117,7 +118,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac .telemetryProtoSchema(schemaStr) .build(); processBeforeTest(configProperties); - DynamicSchema telemetrySchema = getDynamicSchema(schemaStr); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -167,7 +168,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac .telemetryTopicFilter(POST_DATA_TELEMETRY_TOPIC) .build(); processBeforeTest(configProperties); - DynamicSchema telemetrySchema = getDynamicSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -230,7 +231,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac .telemetryProtoSchema(schemaStr) .build(); processBeforeTest(configProperties); - DynamicSchema telemetrySchema = getDynamicSchema(schemaStr); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -458,19 +459,20 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac assertFalse(callback.isPubAckReceived()); } - private DynamicSchema getDynamicSchema(String deviceTelemetryProtoSchema) { + private DynamicSchema getDynamicSchema() { DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration); MqttDeviceProfileTransportConfiguration mqttTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(deviceTelemetryProtoSchema); - return protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); } private DynamicMessage getDefaultDynamicMessage() { - DynamicSchema telemetrySchema = getDynamicSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/common/data/pom.xml b/common/data/pom.xml index decbba55aa..fd927c13d8 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -108,6 +108,10 @@ de.ruedigermoeller fst + + com.google.protobuf + protobuf-java-util + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java new file mode 100644 index 0000000000..8951222ebd --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java @@ -0,0 +1,301 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.github.os72.protobuf.dynamic.EnumDefinition; +import com.github.os72.protobuf.dynamic.MessageDefinition; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import com.squareup.wire.Syntax; +import com.squareup.wire.schema.Field; +import com.squareup.wire.schema.Location; +import com.squareup.wire.schema.internal.parser.EnumConstantElement; +import com.squareup.wire.schema.internal.parser.EnumElement; +import com.squareup.wire.schema.internal.parser.FieldElement; +import com.squareup.wire.schema.internal.parser.MessageElement; +import com.squareup.wire.schema.internal.parser.OneOfElement; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; +import com.squareup.wire.schema.internal.parser.ProtoParser; +import com.squareup.wire.schema.internal.parser.TypeElement; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +public class DynamicProtoUtils { + + public static final Location LOCATION = new Location("", "", -1, -1); + public static final String PROTO_3_SYNTAX = "proto3"; + + public static Descriptors.Descriptor getDescriptor(String protoSchema, String schemaName) { + try { + DynamicMessage.Builder builder = getDynamicMessageBuilder(protoSchema, schemaName); + return builder.getDescriptorForType(); + } catch (Exception e) { + log.warn("Failed to get Message Descriptor due to {}", e.getMessage()); + return null; + } + } + + public static DynamicMessage.Builder getDynamicMessageBuilder(String protoSchema, String schemaName) { + ProtoFileElement protoFileElement = getProtoFileElement(protoSchema); + DynamicSchema dynamicSchema = getDynamicSchema(protoFileElement, schemaName); + String lastMsgName = getMessageTypes(protoFileElement.getTypes()).stream() + .map(MessageElement::getName).reduce((previous, last) -> last).get(); + return dynamicSchema.newMessageBuilder(lastMsgName); + } + + public static DynamicSchema getDynamicSchema(ProtoFileElement protoFileElement, String schemaName) { + DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); + schemaBuilder.setName(schemaName); + schemaBuilder.setSyntax(PROTO_3_SYNTAX); + schemaBuilder.setPackage(StringUtils.isNotEmpty(protoFileElement.getPackageName()) ? + protoFileElement.getPackageName() : schemaName.toLowerCase()); + List types = protoFileElement.getTypes(); + List messageTypes = getMessageTypes(types); + + if (!messageTypes.isEmpty()) { + List enumTypes = getEnumElements(types); + if (!enumTypes.isEmpty()) { + enumTypes.forEach(enumElement -> { + EnumDefinition enumDefinition = getEnumDefinition(enumElement); + schemaBuilder.addEnumDefinition(enumDefinition); + }); + } + List messageDefinitions = getMessageDefinitions(messageTypes); + messageDefinitions.forEach(schemaBuilder::addMessageDefinition); + try { + return schemaBuilder.build(); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException("Failed to create dynamic schema due to: " + e.getMessage()); + } + } else { + throw new RuntimeException("Failed to get Dynamic Schema! Message types is empty for schema:" + schemaName); + } + } + + public static ProtoFileElement getProtoFileElement(String protoSchema) { + return new ProtoParser(LOCATION, protoSchema.toCharArray()).readProtoFile(); + } + + public static String dynamicMsgToJson(Descriptors.Descriptor descriptor, byte[] payload) throws InvalidProtocolBufferException { + DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, payload); + return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage); + } + + public static DynamicMessage jsonToDynamicMessage(DynamicMessage.Builder builder, String payload) throws InvalidProtocolBufferException { + JsonFormat.parser().ignoringUnknownFields().merge(payload, builder); + return builder.build(); + } + + private static List getMessageTypes(List types) { + return types.stream() + .filter(typeElement -> typeElement instanceof MessageElement) + .map(typeElement -> (MessageElement) typeElement) + .collect(Collectors.toList()); + } + + private static List getEnumElements(List types) { + return types.stream() + .filter(typeElement -> typeElement instanceof EnumElement) + .map(typeElement -> (EnumElement) typeElement) + .collect(Collectors.toList()); + } + + private static List getMessageDefinitions(List messageElementsList) { + if (!messageElementsList.isEmpty()) { + List messageDefinitions = new ArrayList<>(); + messageElementsList.forEach(messageElement -> { + MessageDefinition.Builder messageDefinitionBuilder = MessageDefinition.newBuilder(messageElement.getName()); + + List nestedTypes = messageElement.getNestedTypes(); + if (!nestedTypes.isEmpty()) { + List nestedEnumTypes = getEnumElements(nestedTypes); + if (!nestedEnumTypes.isEmpty()) { + nestedEnumTypes.forEach(enumElement -> { + EnumDefinition nestedEnumDefinition = getEnumDefinition(enumElement); + messageDefinitionBuilder.addEnumDefinition(nestedEnumDefinition); + }); + } + List nestedMessageTypes = getMessageTypes(nestedTypes); + List nestedMessageDefinitions = getMessageDefinitions(nestedMessageTypes); + nestedMessageDefinitions.forEach(messageDefinitionBuilder::addMessageDefinition); + } + List messageElementFields = messageElement.getFields(); + List oneOfs = messageElement.getOneOfs(); + if (!oneOfs.isEmpty()) { + for (OneOfElement oneOfelement : oneOfs) { + MessageDefinition.OneofBuilder oneofBuilder = messageDefinitionBuilder.addOneof(oneOfelement.getName()); + addMessageFieldsToTheOneOfDefinition(oneOfelement.getFields(), oneofBuilder); + } + } + if (!messageElementFields.isEmpty()) { + addMessageFieldsToTheMessageDefinition(messageElementFields, messageDefinitionBuilder); + } + messageDefinitions.add(messageDefinitionBuilder.build()); + }); + return messageDefinitions; + } else { + return Collections.emptyList(); + } + } + + private static EnumDefinition getEnumDefinition(EnumElement enumElement) { + List enumElementTypeConstants = enumElement.getConstants(); + EnumDefinition.Builder enumDefinitionBuilder = EnumDefinition.newBuilder(enumElement.getName()); + if (!enumElementTypeConstants.isEmpty()) { + enumElementTypeConstants.forEach(constantElement -> enumDefinitionBuilder.addValue(constantElement.getName(), constantElement.getTag())); + } + return enumDefinitionBuilder.build(); + } + + + private static void addMessageFieldsToTheMessageDefinition(List messageElementFields, MessageDefinition.Builder messageDefinitionBuilder) { + messageElementFields.forEach(fieldElement -> { + String labelStr = null; + if (fieldElement.getLabel() != null) { + labelStr = fieldElement.getLabel().name().toLowerCase(); + } + messageDefinitionBuilder.addField( + labelStr, + fieldElement.getType(), + fieldElement.getName(), + fieldElement.getTag()); + }); + } + + private static void addMessageFieldsToTheOneOfDefinition(List oneOfsElementFields, MessageDefinition.OneofBuilder oneofBuilder) { + oneOfsElementFields.forEach(fieldElement -> oneofBuilder.addField( + fieldElement.getType(), + fieldElement.getName(), + fieldElement.getTag())); + oneofBuilder.msgDefBuilder(); + } + + // validation + + public static void validateProtoSchema(String schema, String schemaName, String exceptionPrefix) throws IllegalArgumentException { + ProtoParser schemaParser = new ProtoParser(LOCATION, schema.toCharArray()); + ProtoFileElement protoFileElement; + try { + protoFileElement = schemaParser.readProtoFile(); + } catch (Exception e) { + throw new IllegalArgumentException(exceptionPrefix + " failed to parse " + schemaName + " due to: " + e.getMessage()); + } + checkProtoFileSyntax(schemaName, protoFileElement); + checkProtoFileCommonSettings(schemaName, protoFileElement.getOptions().isEmpty(), " Schema options don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, protoFileElement.getPublicImports().isEmpty(), " Schema public imports don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, protoFileElement.getImports().isEmpty(), " Schema imports don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, protoFileElement.getExtendDeclarations().isEmpty(), " Schema extend declarations don't support!", exceptionPrefix); + checkTypeElements(schemaName, protoFileElement, exceptionPrefix); + } + + private static void checkProtoFileSyntax(String schemaName, ProtoFileElement protoFileElement) { + if (protoFileElement.getSyntax() == null || !protoFileElement.getSyntax().equals(Syntax.PROTO_3)) { + throw new IllegalArgumentException("[Transport Configuration] invalid schema syntax: " + protoFileElement.getSyntax() + + " for " + schemaName + " provided! Only " + Syntax.PROTO_3 + " allowed!"); + } + } + + private static void checkProtoFileCommonSettings(String schemaName, boolean isEmptySettings, String invalidSettingsMessage, String exceptionPrefix) { + if (!isEmptySettings) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + invalidSettingsMessage); + } + } + + private static void checkTypeElements(String schemaName, ProtoFileElement protoFileElement, String exceptionPrefix) { + List types = protoFileElement.getTypes(); + if (!types.isEmpty()) { + if (types.stream().noneMatch(typeElement -> typeElement instanceof MessageElement)) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " At least one Message definition should exists!"); + } else { + checkEnumElements(schemaName, getEnumElements(types), exceptionPrefix); + checkMessageElements(schemaName, getMessageTypes(types), exceptionPrefix); + } + } else { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Type elements is empty!"); + } + } + + private static void checkFieldElements(String schemaName, List fieldElements, String exceptionPrefix) { + if (!fieldElements.isEmpty()) { + boolean hasRequiredLabel = fieldElements.stream().anyMatch(fieldElement -> { + Field.Label label = fieldElement.getLabel(); + return label != null && label.equals(Field.Label.REQUIRED); + }); + if (hasRequiredLabel) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Required labels are not supported!"); + } + boolean hasDefaultValue = fieldElements.stream().anyMatch(fieldElement -> fieldElement.getDefaultValue() != null); + if (hasDefaultValue) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Default values are not supported!"); + } + } + } + + private static void checkEnumElements(String schemaName, List enumTypes, String exceptionPrefix) { + if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getNestedTypes().isEmpty())) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Nested types in Enum definitions are not supported!"); + } + if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getOptions().isEmpty())) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Enum definitions options are not supported!"); + } + } + + private static void checkMessageElements(String schemaName, List messageElementsList, String exceptionPrefix) { + if (!messageElementsList.isEmpty()) { + messageElementsList.forEach(messageElement -> { + checkProtoFileCommonSettings(schemaName, messageElement.getGroups().isEmpty(), + " Message definition groups don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, messageElement.getOptions().isEmpty(), + " Message definition options don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, messageElement.getExtensions().isEmpty(), + " Message definition extensions don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, messageElement.getReserveds().isEmpty(), + " Message definition reserved elements don't support!", exceptionPrefix); + checkFieldElements(schemaName, messageElement.getFields(), exceptionPrefix); + List oneOfs = messageElement.getOneOfs(); + if (!oneOfs.isEmpty()) { + oneOfs.forEach(oneOfElement -> { + checkProtoFileCommonSettings(schemaName, oneOfElement.getGroups().isEmpty(), + " OneOf definition groups don't support!", exceptionPrefix); + checkFieldElements(schemaName, oneOfElement.getFields(), exceptionPrefix); + }); + } + List nestedTypes = messageElement.getNestedTypes(); + if (!nestedTypes.isEmpty()) { + List nestedEnumTypes = getEnumElements(nestedTypes); + if (!nestedEnumTypes.isEmpty()) { + checkEnumElements(schemaName, nestedEnumTypes, exceptionPrefix); + } + List nestedMessageTypes = getMessageTypes(nestedTypes); + checkMessageElements(schemaName, nestedMessageTypes, exceptionPrefix); + } + }); + } + } + + public static String invalidSchemaProvidedMessage(String schemaName, String exceptionPrefix) { + return exceptionPrefix + " invalid " + schemaName + " provided!"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java index 409570887c..196069ff5f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java @@ -15,29 +15,15 @@ */ package org.thingsboard.server.common.data.device.profile; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.github.os72.protobuf.dynamic.EnumDefinition; -import com.github.os72.protobuf.dynamic.MessageDefinition; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.squareup.wire.schema.Location; -import com.squareup.wire.schema.internal.parser.EnumConstantElement; -import com.squareup.wire.schema.internal.parser.EnumElement; -import com.squareup.wire.schema.internal.parser.FieldElement; -import com.squareup.wire.schema.internal.parser.MessageElement; -import com.squareup.wire.schema.internal.parser.OneOfElement; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; -import com.squareup.wire.schema.internal.parser.ProtoParser; -import com.squareup.wire.schema.internal.parser.TypeElement; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.DynamicProtoUtils; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TransportPayloadType; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - @Slf4j @Data public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeConfiguration { @@ -63,23 +49,23 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC } public Descriptors.Descriptor getTelemetryDynamicMessageDescriptor(String deviceTelemetryProtoSchema) { - return getDescriptor(deviceTelemetryProtoSchema, TELEMETRY_PROTO_SCHEMA); + return DynamicProtoUtils.getDescriptor(deviceTelemetryProtoSchema, TELEMETRY_PROTO_SCHEMA); } public Descriptors.Descriptor getAttributesDynamicMessageDescriptor(String deviceAttributesProtoSchema) { - return getDescriptor(deviceAttributesProtoSchema, ATTRIBUTES_PROTO_SCHEMA); + return DynamicProtoUtils.getDescriptor(deviceAttributesProtoSchema, ATTRIBUTES_PROTO_SCHEMA); } public Descriptors.Descriptor getRpcResponseDynamicMessageDescriptor(String deviceRpcResponseProtoSchema) { - return getDescriptor(deviceRpcResponseProtoSchema, RPC_RESPONSE_PROTO_SCHEMA); + return DynamicProtoUtils.getDescriptor(deviceRpcResponseProtoSchema, RPC_RESPONSE_PROTO_SCHEMA); } public DynamicMessage.Builder getRpcRequestDynamicMessageBuilder(String deviceRpcRequestProtoSchema) { - return getDynamicMessageBuilder(deviceRpcRequestProtoSchema, RPC_REQUEST_PROTO_SCHEMA); + return DynamicProtoUtils.getDynamicMessageBuilder(deviceRpcRequestProtoSchema, RPC_REQUEST_PROTO_SCHEMA); } public String getDeviceRpcResponseProtoSchema() { - if (!isEmptyStr(deviceRpcResponseProtoSchema)) { + if (StringUtils.isNotEmpty(deviceRpcResponseProtoSchema)) { return deviceRpcResponseProtoSchema; } else { return "syntax =\"proto3\";\n" + @@ -92,7 +78,7 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC } public String getDeviceRpcRequestProtoSchema() { - if (!isEmptyStr(deviceRpcRequestProtoSchema)) { + if (StringUtils.isNotEmpty(deviceRpcRequestProtoSchema)) { return deviceRpcRequestProtoSchema; } else { return "syntax =\"proto3\";\n" + @@ -106,143 +92,4 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC } } - private Descriptors.Descriptor getDescriptor(String protoSchema, String schemaName) { - try { - DynamicMessage.Builder builder = getDynamicMessageBuilder(protoSchema, schemaName); - return builder.getDescriptorForType(); - } catch (Exception e) { - log.warn("Failed to get Message Descriptor due to {}", e.getMessage()); - return null; - } - } - - public DynamicMessage.Builder getDynamicMessageBuilder(String protoSchema, String schemaName) { - ProtoFileElement protoFileElement = getTransportProtoSchema(protoSchema); - DynamicSchema dynamicSchema = getDynamicSchema(protoFileElement, schemaName); - String lastMsgName = getMessageTypes(protoFileElement.getTypes()).stream() - .map(MessageElement::getName).reduce((previous, last) -> last).get(); - return dynamicSchema.newMessageBuilder(lastMsgName); - } - - public DynamicSchema getDynamicSchema(ProtoFileElement protoFileElement, String schemaName) { - DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); - schemaBuilder.setName(schemaName); - schemaBuilder.setSyntax(PROTO_3_SYNTAX); - schemaBuilder.setPackage(!isEmptyStr(protoFileElement.getPackageName()) ? - protoFileElement.getPackageName() : schemaName.toLowerCase()); - List types = protoFileElement.getTypes(); - List messageTypes = getMessageTypes(types); - - if (!messageTypes.isEmpty()) { - List enumTypes = getEnumElements(types); - if (!enumTypes.isEmpty()) { - enumTypes.forEach(enumElement -> { - EnumDefinition enumDefinition = getEnumDefinition(enumElement); - schemaBuilder.addEnumDefinition(enumDefinition); - }); - } - List messageDefinitions = getMessageDefinitions(messageTypes); - messageDefinitions.forEach(schemaBuilder::addMessageDefinition); - try { - return schemaBuilder.build(); - } catch (Descriptors.DescriptorValidationException e) { - throw new RuntimeException("Failed to create dynamic schema due to: " + e.getMessage()); - } - } else { - throw new RuntimeException("Failed to get Dynamic Schema! Message types is empty for schema:" + schemaName); - } - } - - public ProtoFileElement getTransportProtoSchema(String protoSchema) { - return new ProtoParser(LOCATION, protoSchema.toCharArray()).readProtoFile(); - } - - private List getMessageTypes(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof MessageElement) - .map(typeElement -> (MessageElement) typeElement) - .collect(Collectors.toList()); - } - - private List getEnumElements(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof EnumElement) - .map(typeElement -> (EnumElement) typeElement) - .collect(Collectors.toList()); - } - - private List getMessageDefinitions(List messageElementsList) { - if (!messageElementsList.isEmpty()) { - List messageDefinitions = new ArrayList<>(); - messageElementsList.forEach(messageElement -> { - MessageDefinition.Builder messageDefinitionBuilder = MessageDefinition.newBuilder(messageElement.getName()); - - List nestedTypes = messageElement.getNestedTypes(); - if (!nestedTypes.isEmpty()) { - List nestedEnumTypes = getEnumElements(nestedTypes); - if (!nestedEnumTypes.isEmpty()) { - nestedEnumTypes.forEach(enumElement -> { - EnumDefinition nestedEnumDefinition = getEnumDefinition(enumElement); - messageDefinitionBuilder.addEnumDefinition(nestedEnumDefinition); - }); - } - List nestedMessageTypes = getMessageTypes(nestedTypes); - List nestedMessageDefinitions = getMessageDefinitions(nestedMessageTypes); - nestedMessageDefinitions.forEach(messageDefinitionBuilder::addMessageDefinition); - } - List messageElementFields = messageElement.getFields(); - List oneOfs = messageElement.getOneOfs(); - if (!oneOfs.isEmpty()) { - for (OneOfElement oneOfelement : oneOfs) { - MessageDefinition.OneofBuilder oneofBuilder = messageDefinitionBuilder.addOneof(oneOfelement.getName()); - addMessageFieldsToTheOneOfDefinition(oneOfelement.getFields(), oneofBuilder); - } - } - if (!messageElementFields.isEmpty()) { - addMessageFieldsToTheMessageDefinition(messageElementFields, messageDefinitionBuilder); - } - messageDefinitions.add(messageDefinitionBuilder.build()); - }); - return messageDefinitions; - } else { - return Collections.emptyList(); - } - } - - private EnumDefinition getEnumDefinition(EnumElement enumElement) { - List enumElementTypeConstants = enumElement.getConstants(); - EnumDefinition.Builder enumDefinitionBuilder = EnumDefinition.newBuilder(enumElement.getName()); - if (!enumElementTypeConstants.isEmpty()) { - enumElementTypeConstants.forEach(constantElement -> enumDefinitionBuilder.addValue(constantElement.getName(), constantElement.getTag())); - } - return enumDefinitionBuilder.build(); - } - - - private void addMessageFieldsToTheMessageDefinition(List messageElementFields, MessageDefinition.Builder messageDefinitionBuilder) { - messageElementFields.forEach(fieldElement -> { - String labelStr = null; - if (fieldElement.getLabel() != null) { - labelStr = fieldElement.getLabel().name().toLowerCase(); - } - messageDefinitionBuilder.addField( - labelStr, - fieldElement.getType(), - fieldElement.getName(), - fieldElement.getTag()); - }); - } - - private void addMessageFieldsToTheOneOfDefinition(List oneOfsElementFields, MessageDefinition.OneofBuilder oneofBuilder) { - oneOfsElementFields.forEach(fieldElement -> oneofBuilder.addField( - fieldElement.getType(), - fieldElement.getName(), - fieldElement.getTag())); - oneofBuilder.msgDefBuilder(); - } - - private boolean isEmptyStr(String str) { - return str == null || "".equals(str); - } - } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java new file mode 100644 index 0000000000..9ae4651cc9 --- /dev/null +++ b/common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java @@ -0,0 +1,169 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class DynamicProtoUtilsTest { + + @Test + public void testProtoSchemaWithMessageNestedTypes() throws Exception { + String schema = "syntax = \"proto3\";\n" + + "\n" + + "package testnested;\n" + + "\n" + + "message Outer {\n" + + " message MiddleAA {\n" + + " message Inner {\n" + + " optional int64 ival = 1;\n" + + " optional bool booly = 2;\n" + + " }\n" + + " Inner inner = 1;\n" + + " }\n" + + " message MiddleBB {\n" + + " message Inner {\n" + + " optional int32 ival = 1;\n" + + " optional bool booly = 2;\n" + + " }\n" + + " Inner inner = 1;\n" + + " }\n" + + " MiddleAA middleAA = 1;\n" + + " MiddleBB middleBB = 2;\n" + + "}"; + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(schema); + DynamicSchema dynamicSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, "test schema with nested types"); + assertNotNull(dynamicSchema); + Set messageTypes = dynamicSchema.getMessageTypes(); + assertEquals(5, messageTypes.size()); + assertTrue(messageTypes.contains("testnested.Outer")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleAA")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleAA.Inner")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleBB")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleBB.Inner")); + + DynamicMessage.Builder middleAAInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); + Descriptors.Descriptor middleAAInnerMsgDescriptor = middleAAInnerMsgBuilder.getDescriptorForType(); + DynamicMessage middleAAInnerMsg = middleAAInnerMsgBuilder + .setField(middleAAInnerMsgDescriptor.findFieldByName("ival"), 1L) + .setField(middleAAInnerMsgDescriptor.findFieldByName("booly"), true) + .build(); + + DynamicMessage.Builder middleAAMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA"); + Descriptors.Descriptor middleAAMsgDescriptor = middleAAMsgBuilder.getDescriptorForType(); + DynamicMessage middleAAMsg = middleAAMsgBuilder + .setField(middleAAMsgDescriptor.findFieldByName("inner"), middleAAInnerMsg) + .build(); + + DynamicMessage.Builder middleBBInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); + Descriptors.Descriptor middleBBInnerMsgDescriptor = middleBBInnerMsgBuilder.getDescriptorForType(); + DynamicMessage middleBBInnerMsg = middleBBInnerMsgBuilder + .setField(middleBBInnerMsgDescriptor.findFieldByName("ival"), 0L) + .setField(middleBBInnerMsgDescriptor.findFieldByName("booly"), false) + .build(); + + DynamicMessage.Builder middleBBMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleBB"); + Descriptors.Descriptor middleBBMsgDescriptor = middleBBMsgBuilder.getDescriptorForType(); + DynamicMessage middleBBMsg = middleBBMsgBuilder + .setField(middleBBMsgDescriptor.findFieldByName("inner"), middleBBInnerMsg) + .build(); + + + DynamicMessage.Builder outerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer"); + Descriptors.Descriptor outerMsgBuilderDescriptor = outerMsgBuilder.getDescriptorForType(); + DynamicMessage outerMsg = outerMsgBuilder + .setField(outerMsgBuilderDescriptor.findFieldByName("middleAA"), middleAAMsg) + .setField(outerMsgBuilderDescriptor.findFieldByName("middleBB"), middleBBMsg) + .build(); + + assertEquals("{\n" + + " \"middleAA\": {\n" + + " \"inner\": {\n" + + " \"ival\": \"1\",\n" + + " \"booly\": true\n" + + " }\n" + + " },\n" + + " \"middleBB\": {\n" + + " \"inner\": {\n" + + " \"ival\": 0,\n" + + " \"booly\": false\n" + + " }\n" + + " }\n" + + "}", DynamicProtoUtils.dynamicMsgToJson(outerMsgBuilderDescriptor, outerMsg.toByteArray())); + } + + @Test + public void testProtoSchemaWithMessageOneOfs() throws Exception { + String schema = "syntax = \"proto3\";\n" + + "\n" + + "package testoneofs;\n" + + "\n" + + "message SubMessage {\n" + + " repeated string name = 1;\n" + + "}\n" + + "\n" + + "message SampleMessage {\n" + + " optional int32 id = 1;\n" + + " oneof testOneOf {\n" + + " string name = 4;\n" + + " SubMessage subMessage = 9;\n" + + " }\n" + + "}"; + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(schema); + DynamicSchema dynamicSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, "test schema with message oneOfs"); + assertNotNull(dynamicSchema); + Set messageTypes = dynamicSchema.getMessageTypes(); + assertEquals(2, messageTypes.size()); + assertTrue(messageTypes.contains("testoneofs.SubMessage")); + assertTrue(messageTypes.contains("testoneofs.SampleMessage")); + + DynamicMessage.Builder sampleMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SampleMessage"); + Descriptors.Descriptor sampleMsgDescriptor = sampleMsgBuilder.getDescriptorForType(); + assertNotNull(sampleMsgDescriptor); + + List fields = sampleMsgDescriptor.getFields(); + assertEquals(3, fields.size()); + DynamicMessage sampleMsg = sampleMsgBuilder + .setField(sampleMsgDescriptor.findFieldByName("name"), "Bob") + .build(); + assertEquals("{\n" + " \"name\": \"Bob\"\n" + "}", DynamicProtoUtils.dynamicMsgToJson(sampleMsgDescriptor, sampleMsg.toByteArray())); + + DynamicMessage.Builder subMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SubMessage"); + Descriptors.Descriptor subMsgDescriptor = subMsgBuilder.getDescriptorForType(); + DynamicMessage subMsg = subMsgBuilder + .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "Alice") + .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "John") + .build(); + + DynamicMessage sampleMsgWithOneOfSubMessage = sampleMsgBuilder.setField(sampleMsgDescriptor.findFieldByName("subMessage"), subMsg).build(); + assertEquals("{\n" + " \"subMessage\": {\n" + " \"name\": [\"Alice\", \"John\"]\n" + " }\n" + "}", + DynamicProtoUtils.dynamicMsgToJson(sampleMsgDescriptor, sampleMsgWithOneOfSubMessage.toByteArray())); + } + +} diff --git a/common/queue/pom.xml b/common/queue/pom.xml index c246ab358f..1ab33a3593 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -116,10 +116,6 @@ com.google.protobuf protobuf-java - - com.google.protobuf - protobuf-java-util - org.apache.curator curator-recipes diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java index 6fe0f29cfc..d7d685bd5a 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java @@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; @@ -184,8 +185,7 @@ public class ProtoConverter { try { JsonElement paramsElement = JSON_PARSER.parse(params); rpcRequestJson.add("params", paramsElement); - JsonFormat.parser().ignoringUnknownFields().merge(GSON.toJson(rpcRequestJson), rpcRequestDynamicMessageBuilder); - DynamicMessage dynamicRpcRequest = rpcRequestDynamicMessageBuilder.build(); + DynamicMessage dynamicRpcRequest = DynamicProtoUtils.jsonToDynamicMessage(rpcRequestDynamicMessageBuilder, GSON.toJson(rpcRequestJson)); return dynamicRpcRequest.toByteArray(); } catch (Exception e) { throw new AdaptorException("Failed to convert ToDeviceRpcRequestMsg to Dynamic Rpc request message due to: ", e); @@ -200,8 +200,7 @@ public class ProtoConverter { } public static String dynamicMsgToJson(byte[] bytes, Descriptors.Descriptor descriptor) throws InvalidProtocolBufferException { - DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, bytes); - return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage); + return DynamicProtoUtils.dynamicMsgToJson(descriptor, bytes); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index 16c9bc4840..84eea112f3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -17,16 +17,6 @@ package org.thingsboard.server.dao.service.validator; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import com.squareup.wire.Syntax; -import com.squareup.wire.schema.Field; -import com.squareup.wire.schema.Location; -import com.squareup.wire.schema.internal.parser.EnumElement; -import com.squareup.wire.schema.internal.parser.FieldElement; -import com.squareup.wire.schema.internal.parser.MessageElement; -import com.squareup.wire.schema.internal.parser.OneOfElement; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; -import com.squareup.wire.schema.internal.parser.ProtoParser; -import com.squareup.wire.schema.internal.parser.TypeElement; import org.eclipse.leshan.core.util.SecurityUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -35,6 +25,7 @@ import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.DashboardInfo; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; @@ -67,16 +58,16 @@ import org.thingsboard.server.dao.tenant.TenantService; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; @Component public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator { - private static final Location LOCATION = new Location("", "", -1, -1); private static final String ATTRIBUTES_PROTO_SCHEMA = "attributes proto schema"; private static final String TELEMETRY_PROTO_SCHEMA = "telemetry proto schema"; private static final String RPC_REQUEST_PROTO_SCHEMA = "rpc request proto schema"; private static final String RPC_RESPONSE_PROTO_SCHEMA = "rpc response proto schema"; + private static final String EXCEPTION_PREFIX = "[Transport Configuration]"; + @Autowired private DeviceProfileDao deviceProfileDao; @Autowired @@ -94,10 +85,6 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator types = protoFileElement.getTypes(); - if (!types.isEmpty()) { - if (types.stream().noneMatch(typeElement -> typeElement instanceof MessageElement)) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " At least one Message definition should exists!"); - } else { - checkEnumElements(schemaName, getEnumElements(types)); - checkMessageElements(schemaName, getMessageTypes(types)); - } - } else { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Type elements is empty!"); - } - } - - private void checkFieldElements(String schemaName, List fieldElements) { - if (!fieldElements.isEmpty()) { - boolean hasRequiredLabel = fieldElements.stream().anyMatch(fieldElement -> { - Field.Label label = fieldElement.getLabel(); - return label != null && label.equals(Field.Label.REQUIRED); - }); - if (hasRequiredLabel) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Required labels are not supported!"); - } - boolean hasDefaultValue = fieldElements.stream().anyMatch(fieldElement -> fieldElement.getDefaultValue() != null); - if (hasDefaultValue) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Default values are not supported!"); - } - } - } - - private void checkEnumElements(String schemaName, List enumTypes) { - if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getNestedTypes().isEmpty())) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Nested types in Enum definitions are not supported!"); - } - if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getOptions().isEmpty())) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Enum definitions options are not supported!"); - } - } - - private void checkMessageElements(String schemaName, List messageElementsList) { - if (!messageElementsList.isEmpty()) { - messageElementsList.forEach(messageElement -> { - checkProtoFileCommonSettings(schemaName, messageElement.getGroups().isEmpty(), - " Message definition groups don't support!"); - checkProtoFileCommonSettings(schemaName, messageElement.getOptions().isEmpty(), - " Message definition options don't support!"); - checkProtoFileCommonSettings(schemaName, messageElement.getExtensions().isEmpty(), - " Message definition extensions don't support!"); - checkProtoFileCommonSettings(schemaName, messageElement.getReserveds().isEmpty(), - " Message definition reserved elements don't support!"); - checkFieldElements(schemaName, messageElement.getFields()); - List oneOfs = messageElement.getOneOfs(); - if (!oneOfs.isEmpty()) { - oneOfs.forEach(oneOfElement -> { - checkProtoFileCommonSettings(schemaName, oneOfElement.getGroups().isEmpty(), - " OneOf definition groups don't support!"); - checkFieldElements(schemaName, oneOfElement.getFields()); - }); - } - List nestedTypes = messageElement.getNestedTypes(); - if (!nestedTypes.isEmpty()) { - List nestedEnumTypes = getEnumElements(nestedTypes); - if (!nestedEnumTypes.isEmpty()) { - checkEnumElements(schemaName, nestedEnumTypes); - } - List nestedMessageTypes = getMessageTypes(nestedTypes); - checkMessageElements(schemaName, nestedMessageTypes); - } - }); - } - } - - private List getMessageTypes(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof MessageElement) - .map(typeElement -> (MessageElement) typeElement) - .collect(Collectors.toList()); - } - - private List getEnumElements(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof EnumElement) - .map(typeElement -> (EnumElement) typeElement) - .collect(Collectors.toList()); - } private void validateTelemetryDynamicMessageFields(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { String deviceTelemetryProtoSchema = protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema(); Descriptors.Descriptor telemetryDynamicMessageDescriptor = protoTransportPayloadTypeConfiguration.getTelemetryDynamicMessageDescriptor(deviceTelemetryProtoSchema); if (telemetryDynamicMessageDescriptor == null) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Failed to get telemetryDynamicMessageDescriptor!"); + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get telemetryDynamicMessageDescriptor!"); } else { List fields = telemetryDynamicMessageDescriptor.getFields(); if (CollectionUtils.isEmpty(fields)) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " " + telemetryDynamicMessageDescriptor.getName() + " fields is empty!"); + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " " + telemetryDynamicMessageDescriptor.getName() + " fields is empty!"); } else if (fields.size() == 2) { Descriptors.FieldDescriptor tsFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("ts"); Descriptors.FieldDescriptor valuesFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("values"); if (tsFieldDescriptor != null && valuesFieldDescriptor != null) { if (!Descriptors.FieldDescriptor.Type.MESSAGE.equals(valuesFieldDescriptor.getType())) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'values' has invalid data type. Only message type is supported!"); + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'values' has invalid data type. Only message type is supported!"); } if (!Descriptors.FieldDescriptor.Type.INT64.equals(tsFieldDescriptor.getType())) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'ts' has invalid data type. Only int64 type is supported!"); + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'ts' has invalid data type. Only int64 type is supported!"); } if (!tsFieldDescriptor.hasOptionalKeyword()) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'ts' has invalid label. Field 'ts' should have optional keyword!"); + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'ts' has invalid label. Field 'ts' should have optional keyword!"); } } } @@ -384,39 +257,39 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator