diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java index 6e0839b354..0b169ae31e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java @@ -16,12 +16,6 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; import org.apache.commons.lang3.RandomStringUtils; import org.junit.After; import org.junit.Assert; @@ -36,10 +30,8 @@ import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; -import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; @@ -47,13 +39,9 @@ import org.thingsboard.server.common.data.security.Authority; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; public abstract class BaseDeviceProfileControllerTest extends AbstractControllerTest { @@ -513,137 +501,6 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController "}", "[Transport Configuration] invalid attributes proto schema provided! OneOf definition groups don't support!"); } - @Test - public void testSaveProtoDeviceProfileWithMessageNestedTypes() throws Exception { - String schema = "syntax = \"proto3\";\n" + - "\n" + - "package testnested;\n" + - "\n" + - "message Outer {\n" + - " message MiddleAA {\n" + - " message Inner {\n" + - " optional int64 ival = 1;\n" + - " optional bool booly = 2;\n" + - " }\n" + - " Inner inner = 1;\n" + - " }\n" + - " message MiddleBB {\n" + - " message Inner {\n" + - " optional int32 ival = 1;\n" + - " optional bool booly = 2;\n" + - " }\n" + - " Inner inner = 1;\n" + - " }\n" + - " MiddleAA middleAA = 1;\n" + - " MiddleBB middleBB = 2;\n" + - "}"; - DynamicSchema dynamicSchema = getDynamicSchema(schema); - assertNotNull(dynamicSchema); - Set messageTypes = dynamicSchema.getMessageTypes(); - assertEquals(5, messageTypes.size()); - assertTrue(messageTypes.contains("testnested.Outer")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleAA")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleAA.Inner")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleBB")); - assertTrue(messageTypes.contains("testnested.Outer.MiddleBB.Inner")); - - DynamicMessage.Builder middleAAInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); - Descriptors.Descriptor middleAAInnerMsgDescriptor = middleAAInnerMsgBuilder.getDescriptorForType(); - DynamicMessage middleAAInnerMsg = middleAAInnerMsgBuilder - .setField(middleAAInnerMsgDescriptor.findFieldByName("ival"), 1L) - .setField(middleAAInnerMsgDescriptor.findFieldByName("booly"), true) - .build(); - - DynamicMessage.Builder middleAAMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA"); - Descriptors.Descriptor middleAAMsgDescriptor = middleAAMsgBuilder.getDescriptorForType(); - DynamicMessage middleAAMsg = middleAAMsgBuilder - .setField(middleAAMsgDescriptor.findFieldByName("inner"), middleAAInnerMsg) - .build(); - - DynamicMessage.Builder middleBBInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); - Descriptors.Descriptor middleBBInnerMsgDescriptor = middleBBInnerMsgBuilder.getDescriptorForType(); - DynamicMessage middleBBInnerMsg = middleBBInnerMsgBuilder - .setField(middleBBInnerMsgDescriptor.findFieldByName("ival"), 0L) - .setField(middleBBInnerMsgDescriptor.findFieldByName("booly"), false) - .build(); - - DynamicMessage.Builder middleBBMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleBB"); - Descriptors.Descriptor middleBBMsgDescriptor = middleBBMsgBuilder.getDescriptorForType(); - DynamicMessage middleBBMsg = middleBBMsgBuilder - .setField(middleBBMsgDescriptor.findFieldByName("inner"), middleBBInnerMsg) - .build(); - - - DynamicMessage.Builder outerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer"); - Descriptors.Descriptor outerMsgBuilderDescriptor = outerMsgBuilder.getDescriptorForType(); - DynamicMessage outerMsg = outerMsgBuilder - .setField(outerMsgBuilderDescriptor.findFieldByName("middleAA"), middleAAMsg) - .setField(outerMsgBuilderDescriptor.findFieldByName("middleBB"), middleBBMsg) - .build(); - - assertEquals("{\n" + - " \"middleAA\": {\n" + - " \"inner\": {\n" + - " \"ival\": \"1\",\n" + - " \"booly\": true\n" + - " }\n" + - " },\n" + - " \"middleBB\": {\n" + - " \"inner\": {\n" + - " \"ival\": 0,\n" + - " \"booly\": false\n" + - " }\n" + - " }\n" + - "}", dynamicMsgToJson(outerMsgBuilderDescriptor, outerMsg.toByteArray())); - } - - @Test - public void testSaveProtoDeviceProfileWithMessageOneOfs() throws Exception { - String schema = "syntax = \"proto3\";\n" + - "\n" + - "package testoneofs;\n" + - "\n" + - "message SubMessage {\n" + - " repeated string name = 1;\n" + - "}\n" + - "\n" + - "message SampleMessage {\n" + - " optional int32 id = 1;\n" + - " oneof testOneOf {\n" + - " string name = 4;\n" + - " SubMessage subMessage = 9;\n" + - " }\n" + - "}"; - DynamicSchema dynamicSchema = getDynamicSchema(schema); - assertNotNull(dynamicSchema); - Set messageTypes = dynamicSchema.getMessageTypes(); - assertEquals(2, messageTypes.size()); - assertTrue(messageTypes.contains("testoneofs.SubMessage")); - assertTrue(messageTypes.contains("testoneofs.SampleMessage")); - - DynamicMessage.Builder sampleMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SampleMessage"); - Descriptors.Descriptor sampleMsgDescriptor = sampleMsgBuilder.getDescriptorForType(); - assertNotNull(sampleMsgDescriptor); - - List fields = sampleMsgDescriptor.getFields(); - assertEquals(3, fields.size()); - DynamicMessage sampleMsg = sampleMsgBuilder - .setField(sampleMsgDescriptor.findFieldByName("name"), "Bob") - .build(); - assertEquals("{\n" + " \"name\": \"Bob\"\n" + "}", dynamicMsgToJson(sampleMsgDescriptor, sampleMsg.toByteArray())); - - DynamicMessage.Builder subMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SubMessage"); - Descriptors.Descriptor subMsgDescriptor = subMsgBuilder.getDescriptorForType(); - DynamicMessage subMsg = subMsgBuilder - .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "Alice") - .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "John") - .build(); - - DynamicMessage sampleMsgWithOneOfSubMessage = sampleMsgBuilder.setField(sampleMsgDescriptor.findFieldByName("subMessage"), subMsg).build(); - assertEquals("{\n" + " \"subMessage\": {\n" + " \"name\": [\"Alice\", \"John\"]\n" + " }\n" + "}", - dynamicMsgToJson(sampleMsgDescriptor, sampleMsgWithOneOfSubMessage.toByteArray())); - } - @Test public void testSaveProtoDeviceProfileWithInvalidTelemetrySchemaTsField() throws Exception { testSaveDeviceProfileWithInvalidProtoSchema("syntax =\"proto3\";\n" + @@ -835,16 +692,6 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController "}", "[Transport Configuration] invalid rpc request proto schema provided! Failed to get field descriptor for field: params!"); } - private DeviceProfile testSaveDeviceProfileWithProtoPayloadType(String schema) throws Exception { - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema, null, null); - MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration); - DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); - DeviceProfile foundDeviceProfile = doGet("/api/deviceProfile/"+savedDeviceProfile.getId().getId().toString(), DeviceProfile.class); - Assert.assertEquals(savedDeviceProfile.getName(), foundDeviceProfile.getName()); - return savedDeviceProfile; - } - private void testSaveDeviceProfileWithInvalidProtoSchema(String schema, String errorMsg) throws Exception { ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema, null, null); MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration); @@ -861,21 +708,4 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController .andExpect(statusReason(containsString(errorMsg))); } - private DynamicSchema getDynamicSchema(String schema) throws Exception { - DeviceProfile deviceProfile = testSaveDeviceProfileWithProtoPayloadType(schema); - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration); - MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttDeviceProfileTransportConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement protoFile = protoTransportPayloadConfiguration.getTransportProtoSchema(schema); - return protoTransportPayloadConfiguration.getDynamicSchema(protoFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - } - - private String dynamicMsgToJson(Descriptors.Descriptor descriptor, byte[] payload) throws InvalidProtocolBufferException { - DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, payload); - return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage); - } - } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/AbstractCoapAttributesRequestProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/AbstractCoapAttributesRequestProtoIntegrationTest.java index 0cbd34ebb7..e331d45d22 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/AbstractCoapAttributesRequestProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/AbstractCoapAttributesRequestProtoIntegrationTest.java @@ -28,6 +28,7 @@ import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -93,8 +94,8 @@ public abstract class AbstractCoapAttributesRequestProtoIntegrationTest extends TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema()); + DynamicSchema attributesSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcProtoIntegrationTest.java index aa79732fde..33a5bafa83 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcProtoIntegrationTest.java @@ -33,6 +33,7 @@ import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -91,8 +92,8 @@ public abstract class AbstractCoapServerSideRpcProtoIntegrationTest extends Abst @Override protected void processOnLoadResponse(CoapResponse response, CoapClient client, Integer observe, CountDownLatch latch) { ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); - ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); - DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); + ProtoFileElement rpcRequestProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcRequestProtoSchema()); + DynamicSchema rpcRequestProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcRequestProtoFileElement, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); byte[] requestPayload = response.getPayload(); DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); @@ -101,8 +102,8 @@ public abstract class AbstractCoapServerSideRpcProtoIntegrationTest extends Abst DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload); Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId"); int requestId = (int) dynamicMessage.getField(requestIdDescriptor); - ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); - DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); + ProtoFileElement rpcResponseProtoSchemaFile = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcResponseProtoSchema()); + DynamicSchema rpcResponseProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); DynamicMessage rpcResponseMsg = rpcResponseBuilder diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesProtoIntegrationTest.java index 0f7e63d1ac..4dc2b1a662 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/AbstractCoapAttributesProtoIntegrationTest.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -47,18 +48,7 @@ public abstract class AbstractCoapAttributesProtoIntegrationTest extends Abstrac @Test public void testPushAttributes() throws Exception { - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - + DynamicSchema attributesSchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); assertNotNull(nestedJsonObjectBuilderDescriptor); @@ -90,18 +80,7 @@ public abstract class AbstractCoapAttributesProtoIntegrationTest extends Abstrac @Test public void testPushAttributesWithExplicitPresenceProtoKeys() throws Exception { - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - + DynamicSchema attributesSchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); assertNotNull(nestedJsonObjectBuilderDescriptor); @@ -127,4 +106,19 @@ public abstract class AbstractCoapAttributesProtoIntegrationTest extends Abstrac processAttributesTest(Arrays.asList("key1", "key5"), postAttributesMsg.toByteArray(), true); } + private DynamicSchema getDynamicSchema() { + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); + CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; + CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); + assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); + DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + String deviceAttributesProtoSchema = protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceAttributesProtoSchema); + return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + } + } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java index 374555ed81..e5e7b83643 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java @@ -24,6 +24,7 @@ import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -58,8 +59,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -127,8 +129,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(schemaStr); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -182,8 +185,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -249,8 +253,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(schemaStr); - DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java index 9c6b97bd59..9ef41782cd 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java @@ -29,6 +29,7 @@ import org.eclipse.paho.client.mqttv3.MqttException; import org.eclipse.paho.client.mqttv3.MqttMessage; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; @@ -515,8 +516,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema()); + DynamicSchema attributesSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 00198666e2..32bfead7ac 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -34,6 +34,7 @@ import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.Assert; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; @@ -414,8 +415,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractM MqttMessage message = new MqttMessage(); if (requestTopic.startsWith(MqttTopics.BASE_DEVICE_API_TOPIC) || requestTopic.startsWith(MqttTopics.BASE_DEVICE_API_TOPIC_V2)) { ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); - ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); - DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); + ProtoFileElement rpcRequestProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcRequestProtoSchema()); + DynamicSchema rpcRequestProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcRequestProtoFileElement, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); byte[] requestPayload = mqttMessage.getPayload(); DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); @@ -427,8 +428,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractM for (Descriptors.FieldDescriptor fieldDescriptor: fields) { assertTrue(dynamicMessage.hasField(fieldDescriptor)); } - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); - DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); + ProtoFileElement rpcResponseProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcResponseProtoSchema()); + DynamicSchema rpcResponseProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcResponseProtoFileElement, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java index ef3ff3c8a1..adf67a5c21 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/AbstractMqttAttributesProtoIntegrationTest.java @@ -22,6 +22,7 @@ import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; @@ -135,8 +136,9 @@ public abstract class AbstractMqttAttributesProtoIntegrationTest extends Abstrac TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA); - return protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + String deviceAttributesProtoSchema = protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceAttributesProtoSchema); + return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); } private DynamicMessage getDefaultDynamicMessage() { diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java index ecb2233332..ac6a30e9b2 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java @@ -25,6 +25,7 @@ import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; @@ -92,7 +93,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac " }\n" + "}"; processBeforeTest("Test Post Telemetry device proto payload", "Test Post Telemetry gateway proto payload", TransportPayloadType.PROTOBUF, POST_DATA_TELEMETRY_TOPIC, null, schemaStr, null, null, null, null, null, DeviceProfileProvisionType.DISABLED, false, false); - DynamicSchema telemetrySchema = getDynamicSchema(schemaStr); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -137,7 +138,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac @Test public void testPushTelemetryWithExplicitPresenceProtoKeys() throws Exception { processBeforeTest("Test Post Telemetry device proto payload", "Test Post Telemetry gateway proto payload", TransportPayloadType.PROTOBUF, POST_DATA_TELEMETRY_TOPIC, null); - DynamicSchema telemetrySchema = getDynamicSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -194,7 +195,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac " }\n" + "}"; processBeforeTest("Test Post Telemetry device proto payload", "Test Post Telemetry gateway proto payload", TransportPayloadType.PROTOBUF, POST_DATA_TELEMETRY_TOPIC, null, schemaStr, null, null, null, null, null, DeviceProfileProvisionType.DISABLED, false, false); - DynamicSchema telemetrySchema = getDynamicSchema(schemaStr); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); @@ -280,19 +281,20 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac assertNotNull(device); } - private DynamicSchema getDynamicSchema(String deviceTelemetryProtoSchema) { + private DynamicSchema getDynamicSchema() { DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration); MqttDeviceProfileTransportConfiguration mqttTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration(); assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(deviceTelemetryProtoSchema); - return protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema"); + String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema(); + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema); + return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA); } private DynamicMessage getDefaultDynamicMessage() { - DynamicSchema telemetrySchema = getDynamicSchema(DEVICE_TELEMETRY_PROTO_SCHEMA); + DynamicSchema telemetrySchema = getDynamicSchema(); DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject"); Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); diff --git a/common/data/pom.xml b/common/data/pom.xml index 450a1cd33d..c9fa24de31 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -104,6 +104,10 @@ io.swagger swagger-annotations + + com.google.protobuf + protobuf-java-util + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java new file mode 100644 index 0000000000..8951222ebd --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java @@ -0,0 +1,301 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.github.os72.protobuf.dynamic.EnumDefinition; +import com.github.os72.protobuf.dynamic.MessageDefinition; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import com.squareup.wire.Syntax; +import com.squareup.wire.schema.Field; +import com.squareup.wire.schema.Location; +import com.squareup.wire.schema.internal.parser.EnumConstantElement; +import com.squareup.wire.schema.internal.parser.EnumElement; +import com.squareup.wire.schema.internal.parser.FieldElement; +import com.squareup.wire.schema.internal.parser.MessageElement; +import com.squareup.wire.schema.internal.parser.OneOfElement; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; +import com.squareup.wire.schema.internal.parser.ProtoParser; +import com.squareup.wire.schema.internal.parser.TypeElement; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +public class DynamicProtoUtils { + + public static final Location LOCATION = new Location("", "", -1, -1); + public static final String PROTO_3_SYNTAX = "proto3"; + + public static Descriptors.Descriptor getDescriptor(String protoSchema, String schemaName) { + try { + DynamicMessage.Builder builder = getDynamicMessageBuilder(protoSchema, schemaName); + return builder.getDescriptorForType(); + } catch (Exception e) { + log.warn("Failed to get Message Descriptor due to {}", e.getMessage()); + return null; + } + } + + public static DynamicMessage.Builder getDynamicMessageBuilder(String protoSchema, String schemaName) { + ProtoFileElement protoFileElement = getProtoFileElement(protoSchema); + DynamicSchema dynamicSchema = getDynamicSchema(protoFileElement, schemaName); + String lastMsgName = getMessageTypes(protoFileElement.getTypes()).stream() + .map(MessageElement::getName).reduce((previous, last) -> last).get(); + return dynamicSchema.newMessageBuilder(lastMsgName); + } + + public static DynamicSchema getDynamicSchema(ProtoFileElement protoFileElement, String schemaName) { + DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); + schemaBuilder.setName(schemaName); + schemaBuilder.setSyntax(PROTO_3_SYNTAX); + schemaBuilder.setPackage(StringUtils.isNotEmpty(protoFileElement.getPackageName()) ? + protoFileElement.getPackageName() : schemaName.toLowerCase()); + List types = protoFileElement.getTypes(); + List messageTypes = getMessageTypes(types); + + if (!messageTypes.isEmpty()) { + List enumTypes = getEnumElements(types); + if (!enumTypes.isEmpty()) { + enumTypes.forEach(enumElement -> { + EnumDefinition enumDefinition = getEnumDefinition(enumElement); + schemaBuilder.addEnumDefinition(enumDefinition); + }); + } + List messageDefinitions = getMessageDefinitions(messageTypes); + messageDefinitions.forEach(schemaBuilder::addMessageDefinition); + try { + return schemaBuilder.build(); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException("Failed to create dynamic schema due to: " + e.getMessage()); + } + } else { + throw new RuntimeException("Failed to get Dynamic Schema! Message types is empty for schema:" + schemaName); + } + } + + public static ProtoFileElement getProtoFileElement(String protoSchema) { + return new ProtoParser(LOCATION, protoSchema.toCharArray()).readProtoFile(); + } + + public static String dynamicMsgToJson(Descriptors.Descriptor descriptor, byte[] payload) throws InvalidProtocolBufferException { + DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, payload); + return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage); + } + + public static DynamicMessage jsonToDynamicMessage(DynamicMessage.Builder builder, String payload) throws InvalidProtocolBufferException { + JsonFormat.parser().ignoringUnknownFields().merge(payload, builder); + return builder.build(); + } + + private static List getMessageTypes(List types) { + return types.stream() + .filter(typeElement -> typeElement instanceof MessageElement) + .map(typeElement -> (MessageElement) typeElement) + .collect(Collectors.toList()); + } + + private static List getEnumElements(List types) { + return types.stream() + .filter(typeElement -> typeElement instanceof EnumElement) + .map(typeElement -> (EnumElement) typeElement) + .collect(Collectors.toList()); + } + + private static List getMessageDefinitions(List messageElementsList) { + if (!messageElementsList.isEmpty()) { + List messageDefinitions = new ArrayList<>(); + messageElementsList.forEach(messageElement -> { + MessageDefinition.Builder messageDefinitionBuilder = MessageDefinition.newBuilder(messageElement.getName()); + + List nestedTypes = messageElement.getNestedTypes(); + if (!nestedTypes.isEmpty()) { + List nestedEnumTypes = getEnumElements(nestedTypes); + if (!nestedEnumTypes.isEmpty()) { + nestedEnumTypes.forEach(enumElement -> { + EnumDefinition nestedEnumDefinition = getEnumDefinition(enumElement); + messageDefinitionBuilder.addEnumDefinition(nestedEnumDefinition); + }); + } + List nestedMessageTypes = getMessageTypes(nestedTypes); + List nestedMessageDefinitions = getMessageDefinitions(nestedMessageTypes); + nestedMessageDefinitions.forEach(messageDefinitionBuilder::addMessageDefinition); + } + List messageElementFields = messageElement.getFields(); + List oneOfs = messageElement.getOneOfs(); + if (!oneOfs.isEmpty()) { + for (OneOfElement oneOfelement : oneOfs) { + MessageDefinition.OneofBuilder oneofBuilder = messageDefinitionBuilder.addOneof(oneOfelement.getName()); + addMessageFieldsToTheOneOfDefinition(oneOfelement.getFields(), oneofBuilder); + } + } + if (!messageElementFields.isEmpty()) { + addMessageFieldsToTheMessageDefinition(messageElementFields, messageDefinitionBuilder); + } + messageDefinitions.add(messageDefinitionBuilder.build()); + }); + return messageDefinitions; + } else { + return Collections.emptyList(); + } + } + + private static EnumDefinition getEnumDefinition(EnumElement enumElement) { + List enumElementTypeConstants = enumElement.getConstants(); + EnumDefinition.Builder enumDefinitionBuilder = EnumDefinition.newBuilder(enumElement.getName()); + if (!enumElementTypeConstants.isEmpty()) { + enumElementTypeConstants.forEach(constantElement -> enumDefinitionBuilder.addValue(constantElement.getName(), constantElement.getTag())); + } + return enumDefinitionBuilder.build(); + } + + + private static void addMessageFieldsToTheMessageDefinition(List messageElementFields, MessageDefinition.Builder messageDefinitionBuilder) { + messageElementFields.forEach(fieldElement -> { + String labelStr = null; + if (fieldElement.getLabel() != null) { + labelStr = fieldElement.getLabel().name().toLowerCase(); + } + messageDefinitionBuilder.addField( + labelStr, + fieldElement.getType(), + fieldElement.getName(), + fieldElement.getTag()); + }); + } + + private static void addMessageFieldsToTheOneOfDefinition(List oneOfsElementFields, MessageDefinition.OneofBuilder oneofBuilder) { + oneOfsElementFields.forEach(fieldElement -> oneofBuilder.addField( + fieldElement.getType(), + fieldElement.getName(), + fieldElement.getTag())); + oneofBuilder.msgDefBuilder(); + } + + // validation + + public static void validateProtoSchema(String schema, String schemaName, String exceptionPrefix) throws IllegalArgumentException { + ProtoParser schemaParser = new ProtoParser(LOCATION, schema.toCharArray()); + ProtoFileElement protoFileElement; + try { + protoFileElement = schemaParser.readProtoFile(); + } catch (Exception e) { + throw new IllegalArgumentException(exceptionPrefix + " failed to parse " + schemaName + " due to: " + e.getMessage()); + } + checkProtoFileSyntax(schemaName, protoFileElement); + checkProtoFileCommonSettings(schemaName, protoFileElement.getOptions().isEmpty(), " Schema options don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, protoFileElement.getPublicImports().isEmpty(), " Schema public imports don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, protoFileElement.getImports().isEmpty(), " Schema imports don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, protoFileElement.getExtendDeclarations().isEmpty(), " Schema extend declarations don't support!", exceptionPrefix); + checkTypeElements(schemaName, protoFileElement, exceptionPrefix); + } + + private static void checkProtoFileSyntax(String schemaName, ProtoFileElement protoFileElement) { + if (protoFileElement.getSyntax() == null || !protoFileElement.getSyntax().equals(Syntax.PROTO_3)) { + throw new IllegalArgumentException("[Transport Configuration] invalid schema syntax: " + protoFileElement.getSyntax() + + " for " + schemaName + " provided! Only " + Syntax.PROTO_3 + " allowed!"); + } + } + + private static void checkProtoFileCommonSettings(String schemaName, boolean isEmptySettings, String invalidSettingsMessage, String exceptionPrefix) { + if (!isEmptySettings) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + invalidSettingsMessage); + } + } + + private static void checkTypeElements(String schemaName, ProtoFileElement protoFileElement, String exceptionPrefix) { + List types = protoFileElement.getTypes(); + if (!types.isEmpty()) { + if (types.stream().noneMatch(typeElement -> typeElement instanceof MessageElement)) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " At least one Message definition should exists!"); + } else { + checkEnumElements(schemaName, getEnumElements(types), exceptionPrefix); + checkMessageElements(schemaName, getMessageTypes(types), exceptionPrefix); + } + } else { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Type elements is empty!"); + } + } + + private static void checkFieldElements(String schemaName, List fieldElements, String exceptionPrefix) { + if (!fieldElements.isEmpty()) { + boolean hasRequiredLabel = fieldElements.stream().anyMatch(fieldElement -> { + Field.Label label = fieldElement.getLabel(); + return label != null && label.equals(Field.Label.REQUIRED); + }); + if (hasRequiredLabel) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Required labels are not supported!"); + } + boolean hasDefaultValue = fieldElements.stream().anyMatch(fieldElement -> fieldElement.getDefaultValue() != null); + if (hasDefaultValue) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Default values are not supported!"); + } + } + } + + private static void checkEnumElements(String schemaName, List enumTypes, String exceptionPrefix) { + if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getNestedTypes().isEmpty())) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Nested types in Enum definitions are not supported!"); + } + if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getOptions().isEmpty())) { + throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Enum definitions options are not supported!"); + } + } + + private static void checkMessageElements(String schemaName, List messageElementsList, String exceptionPrefix) { + if (!messageElementsList.isEmpty()) { + messageElementsList.forEach(messageElement -> { + checkProtoFileCommonSettings(schemaName, messageElement.getGroups().isEmpty(), + " Message definition groups don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, messageElement.getOptions().isEmpty(), + " Message definition options don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, messageElement.getExtensions().isEmpty(), + " Message definition extensions don't support!", exceptionPrefix); + checkProtoFileCommonSettings(schemaName, messageElement.getReserveds().isEmpty(), + " Message definition reserved elements don't support!", exceptionPrefix); + checkFieldElements(schemaName, messageElement.getFields(), exceptionPrefix); + List oneOfs = messageElement.getOneOfs(); + if (!oneOfs.isEmpty()) { + oneOfs.forEach(oneOfElement -> { + checkProtoFileCommonSettings(schemaName, oneOfElement.getGroups().isEmpty(), + " OneOf definition groups don't support!", exceptionPrefix); + checkFieldElements(schemaName, oneOfElement.getFields(), exceptionPrefix); + }); + } + List nestedTypes = messageElement.getNestedTypes(); + if (!nestedTypes.isEmpty()) { + List nestedEnumTypes = getEnumElements(nestedTypes); + if (!nestedEnumTypes.isEmpty()) { + checkEnumElements(schemaName, nestedEnumTypes, exceptionPrefix); + } + List nestedMessageTypes = getMessageTypes(nestedTypes); + checkMessageElements(schemaName, nestedMessageTypes, exceptionPrefix); + } + }); + } + } + + public static String invalidSchemaProvidedMessage(String schemaName, String exceptionPrefix) { + return exceptionPrefix + " invalid " + schemaName + " provided!"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java index 409570887c..196069ff5f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java @@ -15,29 +15,15 @@ */ package org.thingsboard.server.common.data.device.profile; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.github.os72.protobuf.dynamic.EnumDefinition; -import com.github.os72.protobuf.dynamic.MessageDefinition; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.squareup.wire.schema.Location; -import com.squareup.wire.schema.internal.parser.EnumConstantElement; -import com.squareup.wire.schema.internal.parser.EnumElement; -import com.squareup.wire.schema.internal.parser.FieldElement; -import com.squareup.wire.schema.internal.parser.MessageElement; -import com.squareup.wire.schema.internal.parser.OneOfElement; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; -import com.squareup.wire.schema.internal.parser.ProtoParser; -import com.squareup.wire.schema.internal.parser.TypeElement; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.DynamicProtoUtils; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TransportPayloadType; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - @Slf4j @Data public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeConfiguration { @@ -63,23 +49,23 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC } public Descriptors.Descriptor getTelemetryDynamicMessageDescriptor(String deviceTelemetryProtoSchema) { - return getDescriptor(deviceTelemetryProtoSchema, TELEMETRY_PROTO_SCHEMA); + return DynamicProtoUtils.getDescriptor(deviceTelemetryProtoSchema, TELEMETRY_PROTO_SCHEMA); } public Descriptors.Descriptor getAttributesDynamicMessageDescriptor(String deviceAttributesProtoSchema) { - return getDescriptor(deviceAttributesProtoSchema, ATTRIBUTES_PROTO_SCHEMA); + return DynamicProtoUtils.getDescriptor(deviceAttributesProtoSchema, ATTRIBUTES_PROTO_SCHEMA); } public Descriptors.Descriptor getRpcResponseDynamicMessageDescriptor(String deviceRpcResponseProtoSchema) { - return getDescriptor(deviceRpcResponseProtoSchema, RPC_RESPONSE_PROTO_SCHEMA); + return DynamicProtoUtils.getDescriptor(deviceRpcResponseProtoSchema, RPC_RESPONSE_PROTO_SCHEMA); } public DynamicMessage.Builder getRpcRequestDynamicMessageBuilder(String deviceRpcRequestProtoSchema) { - return getDynamicMessageBuilder(deviceRpcRequestProtoSchema, RPC_REQUEST_PROTO_SCHEMA); + return DynamicProtoUtils.getDynamicMessageBuilder(deviceRpcRequestProtoSchema, RPC_REQUEST_PROTO_SCHEMA); } public String getDeviceRpcResponseProtoSchema() { - if (!isEmptyStr(deviceRpcResponseProtoSchema)) { + if (StringUtils.isNotEmpty(deviceRpcResponseProtoSchema)) { return deviceRpcResponseProtoSchema; } else { return "syntax =\"proto3\";\n" + @@ -92,7 +78,7 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC } public String getDeviceRpcRequestProtoSchema() { - if (!isEmptyStr(deviceRpcRequestProtoSchema)) { + if (StringUtils.isNotEmpty(deviceRpcRequestProtoSchema)) { return deviceRpcRequestProtoSchema; } else { return "syntax =\"proto3\";\n" + @@ -106,143 +92,4 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC } } - private Descriptors.Descriptor getDescriptor(String protoSchema, String schemaName) { - try { - DynamicMessage.Builder builder = getDynamicMessageBuilder(protoSchema, schemaName); - return builder.getDescriptorForType(); - } catch (Exception e) { - log.warn("Failed to get Message Descriptor due to {}", e.getMessage()); - return null; - } - } - - public DynamicMessage.Builder getDynamicMessageBuilder(String protoSchema, String schemaName) { - ProtoFileElement protoFileElement = getTransportProtoSchema(protoSchema); - DynamicSchema dynamicSchema = getDynamicSchema(protoFileElement, schemaName); - String lastMsgName = getMessageTypes(protoFileElement.getTypes()).stream() - .map(MessageElement::getName).reduce((previous, last) -> last).get(); - return dynamicSchema.newMessageBuilder(lastMsgName); - } - - public DynamicSchema getDynamicSchema(ProtoFileElement protoFileElement, String schemaName) { - DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); - schemaBuilder.setName(schemaName); - schemaBuilder.setSyntax(PROTO_3_SYNTAX); - schemaBuilder.setPackage(!isEmptyStr(protoFileElement.getPackageName()) ? - protoFileElement.getPackageName() : schemaName.toLowerCase()); - List types = protoFileElement.getTypes(); - List messageTypes = getMessageTypes(types); - - if (!messageTypes.isEmpty()) { - List enumTypes = getEnumElements(types); - if (!enumTypes.isEmpty()) { - enumTypes.forEach(enumElement -> { - EnumDefinition enumDefinition = getEnumDefinition(enumElement); - schemaBuilder.addEnumDefinition(enumDefinition); - }); - } - List messageDefinitions = getMessageDefinitions(messageTypes); - messageDefinitions.forEach(schemaBuilder::addMessageDefinition); - try { - return schemaBuilder.build(); - } catch (Descriptors.DescriptorValidationException e) { - throw new RuntimeException("Failed to create dynamic schema due to: " + e.getMessage()); - } - } else { - throw new RuntimeException("Failed to get Dynamic Schema! Message types is empty for schema:" + schemaName); - } - } - - public ProtoFileElement getTransportProtoSchema(String protoSchema) { - return new ProtoParser(LOCATION, protoSchema.toCharArray()).readProtoFile(); - } - - private List getMessageTypes(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof MessageElement) - .map(typeElement -> (MessageElement) typeElement) - .collect(Collectors.toList()); - } - - private List getEnumElements(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof EnumElement) - .map(typeElement -> (EnumElement) typeElement) - .collect(Collectors.toList()); - } - - private List getMessageDefinitions(List messageElementsList) { - if (!messageElementsList.isEmpty()) { - List messageDefinitions = new ArrayList<>(); - messageElementsList.forEach(messageElement -> { - MessageDefinition.Builder messageDefinitionBuilder = MessageDefinition.newBuilder(messageElement.getName()); - - List nestedTypes = messageElement.getNestedTypes(); - if (!nestedTypes.isEmpty()) { - List nestedEnumTypes = getEnumElements(nestedTypes); - if (!nestedEnumTypes.isEmpty()) { - nestedEnumTypes.forEach(enumElement -> { - EnumDefinition nestedEnumDefinition = getEnumDefinition(enumElement); - messageDefinitionBuilder.addEnumDefinition(nestedEnumDefinition); - }); - } - List nestedMessageTypes = getMessageTypes(nestedTypes); - List nestedMessageDefinitions = getMessageDefinitions(nestedMessageTypes); - nestedMessageDefinitions.forEach(messageDefinitionBuilder::addMessageDefinition); - } - List messageElementFields = messageElement.getFields(); - List oneOfs = messageElement.getOneOfs(); - if (!oneOfs.isEmpty()) { - for (OneOfElement oneOfelement : oneOfs) { - MessageDefinition.OneofBuilder oneofBuilder = messageDefinitionBuilder.addOneof(oneOfelement.getName()); - addMessageFieldsToTheOneOfDefinition(oneOfelement.getFields(), oneofBuilder); - } - } - if (!messageElementFields.isEmpty()) { - addMessageFieldsToTheMessageDefinition(messageElementFields, messageDefinitionBuilder); - } - messageDefinitions.add(messageDefinitionBuilder.build()); - }); - return messageDefinitions; - } else { - return Collections.emptyList(); - } - } - - private EnumDefinition getEnumDefinition(EnumElement enumElement) { - List enumElementTypeConstants = enumElement.getConstants(); - EnumDefinition.Builder enumDefinitionBuilder = EnumDefinition.newBuilder(enumElement.getName()); - if (!enumElementTypeConstants.isEmpty()) { - enumElementTypeConstants.forEach(constantElement -> enumDefinitionBuilder.addValue(constantElement.getName(), constantElement.getTag())); - } - return enumDefinitionBuilder.build(); - } - - - private void addMessageFieldsToTheMessageDefinition(List messageElementFields, MessageDefinition.Builder messageDefinitionBuilder) { - messageElementFields.forEach(fieldElement -> { - String labelStr = null; - if (fieldElement.getLabel() != null) { - labelStr = fieldElement.getLabel().name().toLowerCase(); - } - messageDefinitionBuilder.addField( - labelStr, - fieldElement.getType(), - fieldElement.getName(), - fieldElement.getTag()); - }); - } - - private void addMessageFieldsToTheOneOfDefinition(List oneOfsElementFields, MessageDefinition.OneofBuilder oneofBuilder) { - oneOfsElementFields.forEach(fieldElement -> oneofBuilder.addField( - fieldElement.getType(), - fieldElement.getName(), - fieldElement.getTag())); - oneofBuilder.msgDefBuilder(); - } - - private boolean isEmptyStr(String str) { - return str == null || "".equals(str); - } - } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java new file mode 100644 index 0000000000..9ae4651cc9 --- /dev/null +++ b/common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java @@ -0,0 +1,169 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class DynamicProtoUtilsTest { + + @Test + public void testProtoSchemaWithMessageNestedTypes() throws Exception { + String schema = "syntax = \"proto3\";\n" + + "\n" + + "package testnested;\n" + + "\n" + + "message Outer {\n" + + " message MiddleAA {\n" + + " message Inner {\n" + + " optional int64 ival = 1;\n" + + " optional bool booly = 2;\n" + + " }\n" + + " Inner inner = 1;\n" + + " }\n" + + " message MiddleBB {\n" + + " message Inner {\n" + + " optional int32 ival = 1;\n" + + " optional bool booly = 2;\n" + + " }\n" + + " Inner inner = 1;\n" + + " }\n" + + " MiddleAA middleAA = 1;\n" + + " MiddleBB middleBB = 2;\n" + + "}"; + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(schema); + DynamicSchema dynamicSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, "test schema with nested types"); + assertNotNull(dynamicSchema); + Set messageTypes = dynamicSchema.getMessageTypes(); + assertEquals(5, messageTypes.size()); + assertTrue(messageTypes.contains("testnested.Outer")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleAA")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleAA.Inner")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleBB")); + assertTrue(messageTypes.contains("testnested.Outer.MiddleBB.Inner")); + + DynamicMessage.Builder middleAAInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); + Descriptors.Descriptor middleAAInnerMsgDescriptor = middleAAInnerMsgBuilder.getDescriptorForType(); + DynamicMessage middleAAInnerMsg = middleAAInnerMsgBuilder + .setField(middleAAInnerMsgDescriptor.findFieldByName("ival"), 1L) + .setField(middleAAInnerMsgDescriptor.findFieldByName("booly"), true) + .build(); + + DynamicMessage.Builder middleAAMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA"); + Descriptors.Descriptor middleAAMsgDescriptor = middleAAMsgBuilder.getDescriptorForType(); + DynamicMessage middleAAMsg = middleAAMsgBuilder + .setField(middleAAMsgDescriptor.findFieldByName("inner"), middleAAInnerMsg) + .build(); + + DynamicMessage.Builder middleBBInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); + Descriptors.Descriptor middleBBInnerMsgDescriptor = middleBBInnerMsgBuilder.getDescriptorForType(); + DynamicMessage middleBBInnerMsg = middleBBInnerMsgBuilder + .setField(middleBBInnerMsgDescriptor.findFieldByName("ival"), 0L) + .setField(middleBBInnerMsgDescriptor.findFieldByName("booly"), false) + .build(); + + DynamicMessage.Builder middleBBMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleBB"); + Descriptors.Descriptor middleBBMsgDescriptor = middleBBMsgBuilder.getDescriptorForType(); + DynamicMessage middleBBMsg = middleBBMsgBuilder + .setField(middleBBMsgDescriptor.findFieldByName("inner"), middleBBInnerMsg) + .build(); + + + DynamicMessage.Builder outerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer"); + Descriptors.Descriptor outerMsgBuilderDescriptor = outerMsgBuilder.getDescriptorForType(); + DynamicMessage outerMsg = outerMsgBuilder + .setField(outerMsgBuilderDescriptor.findFieldByName("middleAA"), middleAAMsg) + .setField(outerMsgBuilderDescriptor.findFieldByName("middleBB"), middleBBMsg) + .build(); + + assertEquals("{\n" + + " \"middleAA\": {\n" + + " \"inner\": {\n" + + " \"ival\": \"1\",\n" + + " \"booly\": true\n" + + " }\n" + + " },\n" + + " \"middleBB\": {\n" + + " \"inner\": {\n" + + " \"ival\": 0,\n" + + " \"booly\": false\n" + + " }\n" + + " }\n" + + "}", DynamicProtoUtils.dynamicMsgToJson(outerMsgBuilderDescriptor, outerMsg.toByteArray())); + } + + @Test + public void testProtoSchemaWithMessageOneOfs() throws Exception { + String schema = "syntax = \"proto3\";\n" + + "\n" + + "package testoneofs;\n" + + "\n" + + "message SubMessage {\n" + + " repeated string name = 1;\n" + + "}\n" + + "\n" + + "message SampleMessage {\n" + + " optional int32 id = 1;\n" + + " oneof testOneOf {\n" + + " string name = 4;\n" + + " SubMessage subMessage = 9;\n" + + " }\n" + + "}"; + ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(schema); + DynamicSchema dynamicSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, "test schema with message oneOfs"); + assertNotNull(dynamicSchema); + Set messageTypes = dynamicSchema.getMessageTypes(); + assertEquals(2, messageTypes.size()); + assertTrue(messageTypes.contains("testoneofs.SubMessage")); + assertTrue(messageTypes.contains("testoneofs.SampleMessage")); + + DynamicMessage.Builder sampleMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SampleMessage"); + Descriptors.Descriptor sampleMsgDescriptor = sampleMsgBuilder.getDescriptorForType(); + assertNotNull(sampleMsgDescriptor); + + List fields = sampleMsgDescriptor.getFields(); + assertEquals(3, fields.size()); + DynamicMessage sampleMsg = sampleMsgBuilder + .setField(sampleMsgDescriptor.findFieldByName("name"), "Bob") + .build(); + assertEquals("{\n" + " \"name\": \"Bob\"\n" + "}", DynamicProtoUtils.dynamicMsgToJson(sampleMsgDescriptor, sampleMsg.toByteArray())); + + DynamicMessage.Builder subMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SubMessage"); + Descriptors.Descriptor subMsgDescriptor = subMsgBuilder.getDescriptorForType(); + DynamicMessage subMsg = subMsgBuilder + .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "Alice") + .addRepeatedField(subMsgDescriptor.findFieldByName("name"), "John") + .build(); + + DynamicMessage sampleMsgWithOneOfSubMessage = sampleMsgBuilder.setField(sampleMsgDescriptor.findFieldByName("subMessage"), subMsg).build(); + assertEquals("{\n" + " \"subMessage\": {\n" + " \"name\": [\"Alice\", \"John\"]\n" + " }\n" + "}", + DynamicProtoUtils.dynamicMsgToJson(sampleMsgDescriptor, sampleMsgWithOneOfSubMessage.toByteArray())); + } + +} diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 8240a4070f..c273323cb7 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -116,10 +116,6 @@ com.google.protobuf protobuf-java - - com.google.protobuf - protobuf-java-util - org.apache.curator curator-recipes diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java index 137ceff337..0fb9a767eb 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java @@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; @@ -184,8 +185,7 @@ public class ProtoConverter { try { JsonElement paramsElement = JSON_PARSER.parse(params); rpcRequestJson.add("params", paramsElement); - JsonFormat.parser().ignoringUnknownFields().merge(GSON.toJson(rpcRequestJson), rpcRequestDynamicMessageBuilder); - DynamicMessage dynamicRpcRequest = rpcRequestDynamicMessageBuilder.build(); + DynamicMessage dynamicRpcRequest = DynamicProtoUtils.jsonToDynamicMessage(rpcRequestDynamicMessageBuilder, GSON.toJson(rpcRequestJson)); return dynamicRpcRequest.toByteArray(); } catch (Exception e) { throw new AdaptorException("Failed to convert ToDeviceRpcRequestMsg to Dynamic Rpc request message due to: ", e); @@ -200,8 +200,7 @@ public class ProtoConverter { } public static String dynamicMsgToJson(byte[] bytes, Descriptors.Descriptor descriptor) throws InvalidProtocolBufferException { - DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, bytes); - return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage); + return DynamicProtoUtils.dynamicMsgToJson(descriptor, bytes); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 9a272b4739..1f7849b7b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -17,18 +17,9 @@ package org.thingsboard.server.dao.device; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import com.squareup.wire.Syntax; -import com.squareup.wire.schema.Field; -import com.squareup.wire.schema.Location; -import com.squareup.wire.schema.internal.parser.EnumElement; -import com.squareup.wire.schema.internal.parser.FieldElement; -import com.squareup.wire.schema.internal.parser.MessageElement; -import com.squareup.wire.schema.internal.parser.OneOfElement; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; -import com.squareup.wire.schema.internal.parser.ProtoParser; -import com.squareup.wire.schema.internal.parser.TypeElement; import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.core.util.SecurityUtil; +import org.thingsboard.server.common.data.DynamicProtoUtils; import org.thingsboard.server.common.data.StringUtils; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; @@ -91,7 +82,6 @@ import java.util.List; import java.util.Set; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; import static org.thingsboard.server.common.data.CacheConstants.DEVICE_PROFILE_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -104,15 +94,11 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D private static final String INCORRECT_DEVICE_PROFILE_ID = "Incorrect deviceProfileId "; private static final String INCORRECT_DEVICE_PROFILE_NAME = "Incorrect deviceProfileName "; - private static final Location LOCATION = new Location("", "", -1, -1); private static final String ATTRIBUTES_PROTO_SCHEMA = "attributes proto schema"; private static final String TELEMETRY_PROTO_SCHEMA = "telemetry proto schema"; private static final String RPC_REQUEST_PROTO_SCHEMA = "rpc request proto schema"; private static final String RPC_RESPONSE_PROTO_SCHEMA = "rpc response proto schema"; - - private static String invalidSchemaProvidedMessage(String schemaName) { - return "[Transport Configuration] invalid " + schemaName + " provided!"; - } + private static final String EXCEPTION_PREFIX = "[Transport Configuration]"; @Autowired(required = false) private QueueService queueService; @@ -386,8 +372,8 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D throw new DataValidationException("Another default device profile is present in scope of current tenant!"); } } - if (!StringUtils.isEmpty(deviceProfile.getDefaultQueueName()) && queueService != null){ - if(!queueService.getQueuesByServiceType(ServiceType.TB_RULE_ENGINE).contains(deviceProfile.getDefaultQueueName())){ + if (!StringUtils.isEmpty(deviceProfile.getDefaultQueueName()) && queueService != null) { + if (!queueService.getQueuesByServiceType(ServiceType.TB_RULE_ENGINE).contains(deviceProfile.getDefaultQueueName())) { throw new DataValidationException("Device profile is referencing to non-existent queue!"); } } @@ -515,270 +501,153 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D private void validateProtoSchemas(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { try { - validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceAttributesProtoSchema(), ATTRIBUTES_PROTO_SCHEMA); - validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema(), TELEMETRY_PROTO_SCHEMA); - validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcRequestProtoSchema(), RPC_REQUEST_PROTO_SCHEMA); - validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcResponseProtoSchema(), RPC_RESPONSE_PROTO_SCHEMA); + DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceAttributesProtoSchema(), ATTRIBUTES_PROTO_SCHEMA, EXCEPTION_PREFIX); + DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema(), TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX); + DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcRequestProtoSchema(), RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX); + DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcResponseProtoSchema(), RPC_RESPONSE_PROTO_SCHEMA, EXCEPTION_PREFIX); } catch (Exception exception) { throw new DataValidationException(exception.getMessage()); } } - private void validateTransportProtoSchema(String schema, String schemaName) throws IllegalArgumentException { - ProtoParser schemaParser = new ProtoParser(LOCATION, schema.toCharArray()); - ProtoFileElement protoFileElement; - try { - protoFileElement = schemaParser.readProtoFile(); - } catch (Exception e) { - throw new IllegalArgumentException("[Transport Configuration] failed to parse " + schemaName + " due to: " + e.getMessage()); - } - checkProtoFileSyntax(schemaName, protoFileElement); - checkProtoFileCommonSettings(schemaName, protoFileElement.getOptions().isEmpty(), " Schema options don't support!"); - checkProtoFileCommonSettings(schemaName, protoFileElement.getPublicImports().isEmpty(), " Schema public imports don't support!"); - checkProtoFileCommonSettings(schemaName, protoFileElement.getImports().isEmpty(), " Schema imports don't support!"); - checkProtoFileCommonSettings(schemaName, protoFileElement.getExtendDeclarations().isEmpty(), " Schema extend declarations don't support!"); - checkTypeElements(schemaName, protoFileElement); - } - - private void checkProtoFileSyntax(String schemaName, ProtoFileElement protoFileElement) { - if (protoFileElement.getSyntax() == null || !protoFileElement.getSyntax().equals(Syntax.PROTO_3)) { - throw new IllegalArgumentException("[Transport Configuration] invalid schema syntax: " + protoFileElement.getSyntax() + - " for " + schemaName + " provided! Only " + Syntax.PROTO_3 + " allowed!"); + private void validateLwm2mServersConfigOfBootstrapForClient(List lwM2MBootstrapServersConfigurations, boolean isBootstrapServerUpdateEnable) { + Set uris = new HashSet<>(); + Set shortServerIds = new HashSet<>(); + for (LwM2MBootstrapServerCredential bootstrapServerCredential : lwM2MBootstrapServersConfigurations) { + AbstractLwM2MBootstrapServerCredential serverConfig = (AbstractLwM2MBootstrapServerCredential) bootstrapServerCredential; + if (!isBootstrapServerUpdateEnable && serverConfig.isBootstrapServerIs()) { + throw new DeviceCredentialsValidationException("Bootstrap config must not include \"Bootstrap Server\". \"Include Bootstrap Server updates\" is " + isBootstrapServerUpdateEnable + "."); + } + String server = serverConfig.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server" + " shortServerId: " + serverConfig.getShortServerId() + ":"; + if (serverConfig.getShortServerId() < 1 || serverConfig.getShortServerId() > 65534) { + throw new DeviceCredentialsValidationException(server + " ShortServerId must not be less than 1 and more than 65534!"); + } + if (!shortServerIds.add(serverConfig.getShortServerId())) { + throw new DeviceCredentialsValidationException(server + " \"Short server Id\" value = " + serverConfig.getShortServerId() + ". This value must be a unique value for all servers!"); + } + String uri = serverConfig.getHost() + ":" + serverConfig.getPort(); + if (!uris.add(uri)) { + throw new DeviceCredentialsValidationException(server + " \"Host + port\" value = " + uri + ". This value must be a unique value for all servers!"); + } + Integer port; + if (LwM2MSecurityMode.NO_SEC.equals(serverConfig.getSecurityMode())) { + port = serverConfig.isBootstrapServerIs() ? 5687 : 5685; + } else { + port = serverConfig.isBootstrapServerIs() ? 5688 : 5686; + } + if (serverConfig.getPort() == null || serverConfig.getPort().intValue() != port) { + throw new DeviceCredentialsValidationException(server + " \"Port\" value = " + serverConfig.getPort() + ". This value for security " + serverConfig.getSecurityMode().name() + " must be " + port + "!"); + } } } - private void checkProtoFileCommonSettings(String schemaName, boolean isEmptySettings, String invalidSettingsMessage) { - if (!isEmptySettings) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + invalidSettingsMessage); + private void validateLwm2mServersCredentialOfBootstrapForClient(LwM2MBootstrapServerCredential bootstrapServerConfig) { + String server; + switch (bootstrapServerConfig.getSecurityMode()) { + case NO_SEC: + case PSK: + break; + case RPK: + RPKLwM2MBootstrapServerCredential rpkServerCredentials = (RPKLwM2MBootstrapServerCredential) bootstrapServerConfig; + server = rpkServerCredentials.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server"; + if (StringUtils.isEmpty(rpkServerCredentials.getServerPublicKey())) { + throw new DeviceCredentialsValidationException(server + " RPK public key must be specified!"); + } + try { + String pubkRpkSever = EncryptionUtil.pubkTrimNewLines(rpkServerCredentials.getServerPublicKey()); + rpkServerCredentials.setServerPublicKey(pubkRpkSever); + SecurityUtil.publicKey.decode(rpkServerCredentials.getDecodedCServerPublicKey()); + } catch (Exception e) { + throw new DeviceCredentialsValidationException(server + " RPK public key must be in standard [RFC7250] and then encoded to Base64 format!"); + } + break; + case X509: + X509LwM2MBootstrapServerCredential x509ServerCredentials = (X509LwM2MBootstrapServerCredential) bootstrapServerConfig; + server = x509ServerCredentials.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server"; + if (StringUtils.isEmpty(x509ServerCredentials.getServerPublicKey())) { + throw new DeviceCredentialsValidationException(server + " X509 certificate must be specified!"); + } + try { + String certServer = EncryptionUtil.certTrimNewLines(x509ServerCredentials.getServerPublicKey()); + x509ServerCredentials.setServerPublicKey(certServer); + SecurityUtil.certificate.decode(x509ServerCredentials.getDecodedCServerPublicKey()); + } catch (Exception e) { + throw new DeviceCredentialsValidationException(server + " X509 certificate must be in DER-encoded X509v3 format and support only EC algorithm and then encoded to Base64 format!"); + } + break; } } - private void checkTypeElements(String schemaName, ProtoFileElement protoFileElement) { - List types = protoFileElement.getTypes(); - if (!types.isEmpty()) { - if (types.stream().noneMatch(typeElement -> typeElement instanceof MessageElement)) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " At least one Message definition should exists!"); - } else { - checkEnumElements(schemaName, getEnumElements(types)); - checkMessageElements(schemaName, getMessageTypes(types)); - } + private void validateTelemetryDynamicMessageFields(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { + String deviceTelemetryProtoSchema = protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema(); + Descriptors.Descriptor telemetryDynamicMessageDescriptor = protoTransportPayloadTypeConfiguration.getTelemetryDynamicMessageDescriptor(deviceTelemetryProtoSchema); + if (telemetryDynamicMessageDescriptor == null) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get telemetryDynamicMessageDescriptor!"); } else { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Type elements is empty!"); + List fields = telemetryDynamicMessageDescriptor.getFields(); + if (CollectionUtils.isEmpty(fields)) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " " + telemetryDynamicMessageDescriptor.getName() + " fields is empty!"); + } else if (fields.size() == 2) { + Descriptors.FieldDescriptor tsFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("ts"); + Descriptors.FieldDescriptor valuesFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("values"); + if (tsFieldDescriptor != null && valuesFieldDescriptor != null) { + if (!Descriptors.FieldDescriptor.Type.MESSAGE.equals(valuesFieldDescriptor.getType())) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'values' has invalid data type. Only message type is supported!"); + } + if (!Descriptors.FieldDescriptor.Type.INT64.equals(tsFieldDescriptor.getType())) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'ts' has invalid data type. Only int64 type is supported!"); + } + if (!tsFieldDescriptor.hasOptionalKeyword()) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'ts' has invalid label. Field 'ts' should have optional keyword!"); + } + } + } } } - private void checkFieldElements(String schemaName, List fieldElements) { - if (!fieldElements.isEmpty()) { - boolean hasRequiredLabel = fieldElements.stream().anyMatch(fieldElement -> { - Field.Label label = fieldElement.getLabel(); - return label != null && label.equals(Field.Label.REQUIRED); - }); - if (hasRequiredLabel) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Required labels are not supported!"); + private void validateRpcRequestDynamicMessageFields(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { + DynamicMessage.Builder rpcRequestDynamicMessageBuilder = protoTransportPayloadTypeConfiguration.getRpcRequestDynamicMessageBuilder(protoTransportPayloadTypeConfiguration.getDeviceRpcRequestProtoSchema()); + Descriptors.Descriptor rpcRequestDynamicMessageDescriptor = rpcRequestDynamicMessageBuilder.getDescriptorForType(); + if (rpcRequestDynamicMessageDescriptor == null) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get rpcRequestDynamicMessageDescriptor!"); + } else { + if (CollectionUtils.isEmpty(rpcRequestDynamicMessageDescriptor.getFields()) || rpcRequestDynamicMessageDescriptor.getFields().size() != 3) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " " + rpcRequestDynamicMessageDescriptor.getName() + " message should always contains 3 fields: method, requestId and params!"); } - boolean hasDefaultValue = fieldElements.stream().anyMatch(fieldElement -> fieldElement.getDefaultValue() != null); - if (hasDefaultValue) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Default values are not supported!"); + Descriptors.FieldDescriptor methodFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("method"); + if (methodFieldDescriptor == null) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get field descriptor for field: method!"); + } else { + if (!Descriptors.FieldDescriptor.Type.STRING.equals(methodFieldDescriptor.getType())) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'method' has invalid data type. Only string type is supported!"); + } + if (methodFieldDescriptor.isRepeated()) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'method' has invalid label!"); + } } - } - } - - private void checkEnumElements(String schemaName, List enumTypes) { - if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getNestedTypes().isEmpty())) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Nested types in Enum definitions are not supported!"); - } - if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getOptions().isEmpty())) { - throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Enum definitions options are not supported!"); - } - } - - private void checkMessageElements(String schemaName, List messageElementsList) { - if (!messageElementsList.isEmpty()) { - messageElementsList.forEach(messageElement -> { - checkProtoFileCommonSettings(schemaName, messageElement.getGroups().isEmpty(), - " Message definition groups don't support!"); - checkProtoFileCommonSettings(schemaName, messageElement.getOptions().isEmpty(), - " Message definition options don't support!"); - checkProtoFileCommonSettings(schemaName, messageElement.getExtensions().isEmpty(), - " Message definition extensions don't support!"); - checkProtoFileCommonSettings(schemaName, messageElement.getReserveds().isEmpty(), - " Message definition reserved elements don't support!"); - checkFieldElements(schemaName, messageElement.getFields()); - List oneOfs = messageElement.getOneOfs(); - if (!oneOfs.isEmpty()) { - oneOfs.forEach(oneOfElement -> { - checkProtoFileCommonSettings(schemaName, oneOfElement.getGroups().isEmpty(), - " OneOf definition groups don't support!"); - checkFieldElements(schemaName, oneOfElement.getFields()); - }); + Descriptors.FieldDescriptor requestIdFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("requestId"); + if (requestIdFieldDescriptor == null) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get field descriptor for field: requestId!"); + } else { + if (!Descriptors.FieldDescriptor.Type.INT32.equals(requestIdFieldDescriptor.getType())) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'requestId' has invalid data type. Only int32 type is supported!"); } - List nestedTypes = messageElement.getNestedTypes(); - if (!nestedTypes.isEmpty()) { - List nestedEnumTypes = getEnumElements(nestedTypes); - if (!nestedEnumTypes.isEmpty()) { - checkEnumElements(schemaName, nestedEnumTypes); - } - List nestedMessageTypes = getMessageTypes(nestedTypes); - checkMessageElements(schemaName, nestedMessageTypes); + if (requestIdFieldDescriptor.isRepeated()) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'requestId' has invalid label!"); } - }); + } + Descriptors.FieldDescriptor paramsFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("params"); + if (paramsFieldDescriptor == null) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get field descriptor for field: params!"); + } else { + if (paramsFieldDescriptor.isRepeated()) { + throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'params' has invalid label!"); + } + } } } - - private List getMessageTypes(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof MessageElement) - .map(typeElement -> (MessageElement) typeElement) - .collect(Collectors.toList()); - } - - private List getEnumElements(List types) { - return types.stream() - .filter(typeElement -> typeElement instanceof EnumElement) - .map(typeElement -> (EnumElement) typeElement) - .collect(Collectors.toList()); - } - }; - private void validateTelemetryDynamicMessageFields(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { - String deviceTelemetryProtoSchema = protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema(); - Descriptors.Descriptor telemetryDynamicMessageDescriptor = protoTransportPayloadTypeConfiguration.getTelemetryDynamicMessageDescriptor(deviceTelemetryProtoSchema); - if (telemetryDynamicMessageDescriptor == null) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Failed to get telemetryDynamicMessageDescriptor!"); - } else { - List fields = telemetryDynamicMessageDescriptor.getFields(); - if (CollectionUtils.isEmpty(fields)) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " " + telemetryDynamicMessageDescriptor.getName() + " fields is empty!"); - } else if (fields.size() == 2) { - Descriptors.FieldDescriptor tsFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("ts"); - Descriptors.FieldDescriptor valuesFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("values"); - if (tsFieldDescriptor != null && valuesFieldDescriptor != null) { - if (!Descriptors.FieldDescriptor.Type.MESSAGE.equals(valuesFieldDescriptor.getType())) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'values' has invalid data type. Only message type is supported!"); - } - if (!Descriptors.FieldDescriptor.Type.INT64.equals(tsFieldDescriptor.getType())) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'ts' has invalid data type. Only int64 type is supported!"); - } - if (!tsFieldDescriptor.hasOptionalKeyword()) { - throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'ts' has invalid label. Field 'ts' should have optional keyword!"); - } - } - } - } - } - - private void validateRpcRequestDynamicMessageFields(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { - DynamicMessage.Builder rpcRequestDynamicMessageBuilder = protoTransportPayloadTypeConfiguration.getRpcRequestDynamicMessageBuilder(protoTransportPayloadTypeConfiguration.getDeviceRpcRequestProtoSchema()); - Descriptors.Descriptor rpcRequestDynamicMessageDescriptor = rpcRequestDynamicMessageBuilder.getDescriptorForType(); - if (rpcRequestDynamicMessageDescriptor == null) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get rpcRequestDynamicMessageDescriptor!"); - } else { - if (CollectionUtils.isEmpty(rpcRequestDynamicMessageDescriptor.getFields()) || rpcRequestDynamicMessageDescriptor.getFields().size() != 3) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " " + rpcRequestDynamicMessageDescriptor.getName() + " message should always contains 3 fields: method, requestId and params!"); - } - Descriptors.FieldDescriptor methodFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("method"); - if (methodFieldDescriptor == null) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get field descriptor for field: method!"); - } else { - if (!Descriptors.FieldDescriptor.Type.STRING.equals(methodFieldDescriptor.getType())) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'method' has invalid data type. Only string type is supported!"); - } - if (methodFieldDescriptor.isRepeated()) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'method' has invalid label!"); - } - } - Descriptors.FieldDescriptor requestIdFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("requestId"); - if (requestIdFieldDescriptor == null) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get field descriptor for field: requestId!"); - } else { - if (!Descriptors.FieldDescriptor.Type.INT32.equals(requestIdFieldDescriptor.getType())) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'requestId' has invalid data type. Only int32 type is supported!"); - } - if (requestIdFieldDescriptor.isRepeated()) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'requestId' has invalid label!"); - } - } - Descriptors.FieldDescriptor paramsFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("params"); - if (paramsFieldDescriptor == null) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get field descriptor for field: params!"); - } else { - if (paramsFieldDescriptor.isRepeated()) { - throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'params' has invalid label!"); - } - } - } - } - - private void validateLwm2mServersConfigOfBootstrapForClient(List lwM2MBootstrapServersConfigurations, boolean isBootstrapServerUpdateEnable) { - Set uris = new HashSet<>(); - Set shortServerIds = new HashSet<>(); - for (LwM2MBootstrapServerCredential bootstrapServerCredential : lwM2MBootstrapServersConfigurations) { - AbstractLwM2MBootstrapServerCredential serverConfig = (AbstractLwM2MBootstrapServerCredential) bootstrapServerCredential; - if (!isBootstrapServerUpdateEnable && serverConfig.isBootstrapServerIs()) { - throw new DeviceCredentialsValidationException("Bootstrap config must not include \"Bootstrap Server\". \"Include Bootstrap Server updates\" is " + isBootstrapServerUpdateEnable + "."); - } - String server = serverConfig.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server" + " shortServerId: " + serverConfig.getShortServerId() + ":"; - if (serverConfig.getShortServerId() < 1 || serverConfig.getShortServerId() > 65534) { - throw new DeviceCredentialsValidationException(server + " ShortServerId must not be less than 1 and more than 65534!"); - } - if (!shortServerIds.add(serverConfig.getShortServerId())) { - throw new DeviceCredentialsValidationException(server + " \"Short server Id\" value = " + serverConfig.getShortServerId() + ". This value must be a unique value for all servers!"); - } - String uri = serverConfig.getHost() + ":" + serverConfig.getPort(); - if (!uris.add(uri)) { - throw new DeviceCredentialsValidationException(server + " \"Host + port\" value = " + uri + ". This value must be a unique value for all servers!"); - } - Integer port; - if (LwM2MSecurityMode.NO_SEC.equals(serverConfig.getSecurityMode())) { - port = serverConfig.isBootstrapServerIs() ? 5687 : 5685; - } else { - port = serverConfig.isBootstrapServerIs() ? 5688 : 5686; - } - if (serverConfig.getPort() == null || serverConfig.getPort().intValue() != port) { - throw new DeviceCredentialsValidationException(server + " \"Port\" value = " + serverConfig.getPort() + ". This value for security " + serverConfig.getSecurityMode().name() + " must be " + port + "!"); - } - } - } - - private void validateLwm2mServersCredentialOfBootstrapForClient(LwM2MBootstrapServerCredential bootstrapServerConfig) { - String server; - switch (bootstrapServerConfig.getSecurityMode()) { - case NO_SEC: - case PSK: - break; - case RPK: - RPKLwM2MBootstrapServerCredential rpkServerCredentials = (RPKLwM2MBootstrapServerCredential) bootstrapServerConfig; - server = rpkServerCredentials.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server"; - if (StringUtils.isEmpty(rpkServerCredentials.getServerPublicKey())) { - throw new DeviceCredentialsValidationException(server + " RPK public key must be specified!"); - } - try { - String pubkRpkSever = EncryptionUtil.pubkTrimNewLines(rpkServerCredentials.getServerPublicKey()); - rpkServerCredentials.setServerPublicKey(pubkRpkSever); - SecurityUtil.publicKey.decode(rpkServerCredentials.getDecodedCServerPublicKey()); - } catch (Exception e) { - throw new DeviceCredentialsValidationException(server + " RPK public key must be in standard [RFC7250] and then encoded to Base64 format!"); - } - break; - case X509: - X509LwM2MBootstrapServerCredential x509ServerCredentials = (X509LwM2MBootstrapServerCredential) bootstrapServerConfig; - server = x509ServerCredentials.isBootstrapServerIs() ? "Bootstrap Server" : "LwM2M Server"; - if (StringUtils.isEmpty(x509ServerCredentials.getServerPublicKey())) { - throw new DeviceCredentialsValidationException(server + " X509 certificate must be specified!"); - } - - try { - String certServer = EncryptionUtil.certTrimNewLines(x509ServerCredentials.getServerPublicKey()); - x509ServerCredentials.setServerPublicKey(certServer); - SecurityUtil.certificate.decode(x509ServerCredentials.getDecodedCServerPublicKey()); - } catch (Exception e) { - throw new DeviceCredentialsValidationException(server + " X509 certificate must be in DER-encoded X509v3 format and support only EC algorithm and then encoded to Base64 format!"); - } - break; - } - } - private PaginatedRemover tenantDeviceProfilesRemover = new PaginatedRemover() {