ShvaykaD
4 years ago
32 changed files with 1160 additions and 654 deletions
@ -0,0 +1,301 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.common.data; |
|||
|
|||
import com.github.os72.protobuf.dynamic.DynamicSchema; |
|||
import com.github.os72.protobuf.dynamic.EnumDefinition; |
|||
import com.github.os72.protobuf.dynamic.MessageDefinition; |
|||
import com.google.protobuf.Descriptors; |
|||
import com.google.protobuf.DynamicMessage; |
|||
import com.google.protobuf.InvalidProtocolBufferException; |
|||
import com.google.protobuf.util.JsonFormat; |
|||
import com.squareup.wire.Syntax; |
|||
import com.squareup.wire.schema.Field; |
|||
import com.squareup.wire.schema.Location; |
|||
import com.squareup.wire.schema.internal.parser.EnumConstantElement; |
|||
import com.squareup.wire.schema.internal.parser.EnumElement; |
|||
import com.squareup.wire.schema.internal.parser.FieldElement; |
|||
import com.squareup.wire.schema.internal.parser.MessageElement; |
|||
import com.squareup.wire.schema.internal.parser.OneOfElement; |
|||
import com.squareup.wire.schema.internal.parser.ProtoFileElement; |
|||
import com.squareup.wire.schema.internal.parser.ProtoParser; |
|||
import com.squareup.wire.schema.internal.parser.TypeElement; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@Slf4j |
|||
public class DynamicProtoUtils { |
|||
|
|||
public static final Location LOCATION = new Location("", "", -1, -1); |
|||
public static final String PROTO_3_SYNTAX = "proto3"; |
|||
|
|||
public static Descriptors.Descriptor getDescriptor(String protoSchema, String schemaName) { |
|||
try { |
|||
DynamicMessage.Builder builder = getDynamicMessageBuilder(protoSchema, schemaName); |
|||
return builder.getDescriptorForType(); |
|||
} catch (Exception e) { |
|||
log.warn("Failed to get Message Descriptor due to {}", e.getMessage()); |
|||
return null; |
|||
} |
|||
} |
|||
|
|||
public static DynamicMessage.Builder getDynamicMessageBuilder(String protoSchema, String schemaName) { |
|||
ProtoFileElement protoFileElement = getProtoFileElement(protoSchema); |
|||
DynamicSchema dynamicSchema = getDynamicSchema(protoFileElement, schemaName); |
|||
String lastMsgName = getMessageTypes(protoFileElement.getTypes()).stream() |
|||
.map(MessageElement::getName).reduce((previous, last) -> last).get(); |
|||
return dynamicSchema.newMessageBuilder(lastMsgName); |
|||
} |
|||
|
|||
public static DynamicSchema getDynamicSchema(ProtoFileElement protoFileElement, String schemaName) { |
|||
DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); |
|||
schemaBuilder.setName(schemaName); |
|||
schemaBuilder.setSyntax(PROTO_3_SYNTAX); |
|||
schemaBuilder.setPackage(StringUtils.isNotEmpty(protoFileElement.getPackageName()) ? |
|||
protoFileElement.getPackageName() : schemaName.toLowerCase()); |
|||
List<TypeElement> types = protoFileElement.getTypes(); |
|||
List<MessageElement> messageTypes = getMessageTypes(types); |
|||
|
|||
if (!messageTypes.isEmpty()) { |
|||
List<EnumElement> enumTypes = getEnumElements(types); |
|||
if (!enumTypes.isEmpty()) { |
|||
enumTypes.forEach(enumElement -> { |
|||
EnumDefinition enumDefinition = getEnumDefinition(enumElement); |
|||
schemaBuilder.addEnumDefinition(enumDefinition); |
|||
}); |
|||
} |
|||
List<MessageDefinition> messageDefinitions = getMessageDefinitions(messageTypes); |
|||
messageDefinitions.forEach(schemaBuilder::addMessageDefinition); |
|||
try { |
|||
return schemaBuilder.build(); |
|||
} catch (Descriptors.DescriptorValidationException e) { |
|||
throw new RuntimeException("Failed to create dynamic schema due to: " + e.getMessage()); |
|||
} |
|||
} else { |
|||
throw new RuntimeException("Failed to get Dynamic Schema! Message types is empty for schema:" + schemaName); |
|||
} |
|||
} |
|||
|
|||
public static ProtoFileElement getProtoFileElement(String protoSchema) { |
|||
return new ProtoParser(LOCATION, protoSchema.toCharArray()).readProtoFile(); |
|||
} |
|||
|
|||
public static String dynamicMsgToJson(Descriptors.Descriptor descriptor, byte[] payload) throws InvalidProtocolBufferException { |
|||
DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, payload); |
|||
return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage); |
|||
} |
|||
|
|||
public static DynamicMessage jsonToDynamicMessage(DynamicMessage.Builder builder, String payload) throws InvalidProtocolBufferException { |
|||
JsonFormat.parser().ignoringUnknownFields().merge(payload, builder); |
|||
return builder.build(); |
|||
} |
|||
|
|||
private static List<MessageElement> getMessageTypes(List<TypeElement> types) { |
|||
return types.stream() |
|||
.filter(typeElement -> typeElement instanceof MessageElement) |
|||
.map(typeElement -> (MessageElement) typeElement) |
|||
.collect(Collectors.toList()); |
|||
} |
|||
|
|||
private static List<EnumElement> getEnumElements(List<TypeElement> types) { |
|||
return types.stream() |
|||
.filter(typeElement -> typeElement instanceof EnumElement) |
|||
.map(typeElement -> (EnumElement) typeElement) |
|||
.collect(Collectors.toList()); |
|||
} |
|||
|
|||
private static List<MessageDefinition> getMessageDefinitions(List<MessageElement> messageElementsList) { |
|||
if (!messageElementsList.isEmpty()) { |
|||
List<MessageDefinition> messageDefinitions = new ArrayList<>(); |
|||
messageElementsList.forEach(messageElement -> { |
|||
MessageDefinition.Builder messageDefinitionBuilder = MessageDefinition.newBuilder(messageElement.getName()); |
|||
|
|||
List<TypeElement> nestedTypes = messageElement.getNestedTypes(); |
|||
if (!nestedTypes.isEmpty()) { |
|||
List<EnumElement> nestedEnumTypes = getEnumElements(nestedTypes); |
|||
if (!nestedEnumTypes.isEmpty()) { |
|||
nestedEnumTypes.forEach(enumElement -> { |
|||
EnumDefinition nestedEnumDefinition = getEnumDefinition(enumElement); |
|||
messageDefinitionBuilder.addEnumDefinition(nestedEnumDefinition); |
|||
}); |
|||
} |
|||
List<MessageElement> nestedMessageTypes = getMessageTypes(nestedTypes); |
|||
List<MessageDefinition> nestedMessageDefinitions = getMessageDefinitions(nestedMessageTypes); |
|||
nestedMessageDefinitions.forEach(messageDefinitionBuilder::addMessageDefinition); |
|||
} |
|||
List<FieldElement> messageElementFields = messageElement.getFields(); |
|||
List<OneOfElement> oneOfs = messageElement.getOneOfs(); |
|||
if (!oneOfs.isEmpty()) { |
|||
for (OneOfElement oneOfelement : oneOfs) { |
|||
MessageDefinition.OneofBuilder oneofBuilder = messageDefinitionBuilder.addOneof(oneOfelement.getName()); |
|||
addMessageFieldsToTheOneOfDefinition(oneOfelement.getFields(), oneofBuilder); |
|||
} |
|||
} |
|||
if (!messageElementFields.isEmpty()) { |
|||
addMessageFieldsToTheMessageDefinition(messageElementFields, messageDefinitionBuilder); |
|||
} |
|||
messageDefinitions.add(messageDefinitionBuilder.build()); |
|||
}); |
|||
return messageDefinitions; |
|||
} else { |
|||
return Collections.emptyList(); |
|||
} |
|||
} |
|||
|
|||
private static EnumDefinition getEnumDefinition(EnumElement enumElement) { |
|||
List<EnumConstantElement> enumElementTypeConstants = enumElement.getConstants(); |
|||
EnumDefinition.Builder enumDefinitionBuilder = EnumDefinition.newBuilder(enumElement.getName()); |
|||
if (!enumElementTypeConstants.isEmpty()) { |
|||
enumElementTypeConstants.forEach(constantElement -> enumDefinitionBuilder.addValue(constantElement.getName(), constantElement.getTag())); |
|||
} |
|||
return enumDefinitionBuilder.build(); |
|||
} |
|||
|
|||
|
|||
private static void addMessageFieldsToTheMessageDefinition(List<FieldElement> messageElementFields, MessageDefinition.Builder messageDefinitionBuilder) { |
|||
messageElementFields.forEach(fieldElement -> { |
|||
String labelStr = null; |
|||
if (fieldElement.getLabel() != null) { |
|||
labelStr = fieldElement.getLabel().name().toLowerCase(); |
|||
} |
|||
messageDefinitionBuilder.addField( |
|||
labelStr, |
|||
fieldElement.getType(), |
|||
fieldElement.getName(), |
|||
fieldElement.getTag()); |
|||
}); |
|||
} |
|||
|
|||
private static void addMessageFieldsToTheOneOfDefinition(List<FieldElement> oneOfsElementFields, MessageDefinition.OneofBuilder oneofBuilder) { |
|||
oneOfsElementFields.forEach(fieldElement -> oneofBuilder.addField( |
|||
fieldElement.getType(), |
|||
fieldElement.getName(), |
|||
fieldElement.getTag())); |
|||
oneofBuilder.msgDefBuilder(); |
|||
} |
|||
|
|||
// validation
|
|||
|
|||
public static void validateProtoSchema(String schema, String schemaName, String exceptionPrefix) throws IllegalArgumentException { |
|||
ProtoParser schemaParser = new ProtoParser(LOCATION, schema.toCharArray()); |
|||
ProtoFileElement protoFileElement; |
|||
try { |
|||
protoFileElement = schemaParser.readProtoFile(); |
|||
} catch (Exception e) { |
|||
throw new IllegalArgumentException(exceptionPrefix + " failed to parse " + schemaName + " due to: " + e.getMessage()); |
|||
} |
|||
checkProtoFileSyntax(schemaName, protoFileElement); |
|||
checkProtoFileCommonSettings(schemaName, protoFileElement.getOptions().isEmpty(), " Schema options don't support!", exceptionPrefix); |
|||
checkProtoFileCommonSettings(schemaName, protoFileElement.getPublicImports().isEmpty(), " Schema public imports don't support!", exceptionPrefix); |
|||
checkProtoFileCommonSettings(schemaName, protoFileElement.getImports().isEmpty(), " Schema imports don't support!", exceptionPrefix); |
|||
checkProtoFileCommonSettings(schemaName, protoFileElement.getExtendDeclarations().isEmpty(), " Schema extend declarations don't support!", exceptionPrefix); |
|||
checkTypeElements(schemaName, protoFileElement, exceptionPrefix); |
|||
} |
|||
|
|||
private static void checkProtoFileSyntax(String schemaName, ProtoFileElement protoFileElement) { |
|||
if (protoFileElement.getSyntax() == null || !protoFileElement.getSyntax().equals(Syntax.PROTO_3)) { |
|||
throw new IllegalArgumentException("[Transport Configuration] invalid schema syntax: " + protoFileElement.getSyntax() + |
|||
" for " + schemaName + " provided! Only " + Syntax.PROTO_3 + " allowed!"); |
|||
} |
|||
} |
|||
|
|||
private static void checkProtoFileCommonSettings(String schemaName, boolean isEmptySettings, String invalidSettingsMessage, String exceptionPrefix) { |
|||
if (!isEmptySettings) { |
|||
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + invalidSettingsMessage); |
|||
} |
|||
} |
|||
|
|||
private static void checkTypeElements(String schemaName, ProtoFileElement protoFileElement, String exceptionPrefix) { |
|||
List<TypeElement> types = protoFileElement.getTypes(); |
|||
if (!types.isEmpty()) { |
|||
if (types.stream().noneMatch(typeElement -> typeElement instanceof MessageElement)) { |
|||
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " At least one Message definition should exists!"); |
|||
} else { |
|||
checkEnumElements(schemaName, getEnumElements(types), exceptionPrefix); |
|||
checkMessageElements(schemaName, getMessageTypes(types), exceptionPrefix); |
|||
} |
|||
} else { |
|||
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Type elements is empty!"); |
|||
} |
|||
} |
|||
|
|||
private static void checkFieldElements(String schemaName, List<FieldElement> fieldElements, String exceptionPrefix) { |
|||
if (!fieldElements.isEmpty()) { |
|||
boolean hasRequiredLabel = fieldElements.stream().anyMatch(fieldElement -> { |
|||
Field.Label label = fieldElement.getLabel(); |
|||
return label != null && label.equals(Field.Label.REQUIRED); |
|||
}); |
|||
if (hasRequiredLabel) { |
|||
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Required labels are not supported!"); |
|||
} |
|||
boolean hasDefaultValue = fieldElements.stream().anyMatch(fieldElement -> fieldElement.getDefaultValue() != null); |
|||
if (hasDefaultValue) { |
|||
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Default values are not supported!"); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private static void checkEnumElements(String schemaName, List<EnumElement> enumTypes, String exceptionPrefix) { |
|||
if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getNestedTypes().isEmpty())) { |
|||
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Nested types in Enum definitions are not supported!"); |
|||
} |
|||
if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getOptions().isEmpty())) { |
|||
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Enum definitions options are not supported!"); |
|||
} |
|||
} |
|||
|
|||
private static void checkMessageElements(String schemaName, List<MessageElement> messageElementsList, String exceptionPrefix) { |
|||
if (!messageElementsList.isEmpty()) { |
|||
messageElementsList.forEach(messageElement -> { |
|||
checkProtoFileCommonSettings(schemaName, messageElement.getGroups().isEmpty(), |
|||
" Message definition groups don't support!", exceptionPrefix); |
|||
checkProtoFileCommonSettings(schemaName, messageElement.getOptions().isEmpty(), |
|||
" Message definition options don't support!", exceptionPrefix); |
|||
checkProtoFileCommonSettings(schemaName, messageElement.getExtensions().isEmpty(), |
|||
" Message definition extensions don't support!", exceptionPrefix); |
|||
checkProtoFileCommonSettings(schemaName, messageElement.getReserveds().isEmpty(), |
|||
" Message definition reserved elements don't support!", exceptionPrefix); |
|||
checkFieldElements(schemaName, messageElement.getFields(), exceptionPrefix); |
|||
List<OneOfElement> oneOfs = messageElement.getOneOfs(); |
|||
if (!oneOfs.isEmpty()) { |
|||
oneOfs.forEach(oneOfElement -> { |
|||
checkProtoFileCommonSettings(schemaName, oneOfElement.getGroups().isEmpty(), |
|||
" OneOf definition groups don't support!", exceptionPrefix); |
|||
checkFieldElements(schemaName, oneOfElement.getFields(), exceptionPrefix); |
|||
}); |
|||
} |
|||
List<TypeElement> nestedTypes = messageElement.getNestedTypes(); |
|||
if (!nestedTypes.isEmpty()) { |
|||
List<EnumElement> nestedEnumTypes = getEnumElements(nestedTypes); |
|||
if (!nestedEnumTypes.isEmpty()) { |
|||
checkEnumElements(schemaName, nestedEnumTypes, exceptionPrefix); |
|||
} |
|||
List<MessageElement> nestedMessageTypes = getMessageTypes(nestedTypes); |
|||
checkMessageElements(schemaName, nestedMessageTypes, exceptionPrefix); |
|||
} |
|||
}); |
|||
} |
|||
} |
|||
|
|||
public static String invalidSchemaProvidedMessage(String schemaName, String exceptionPrefix) { |
|||
return exceptionPrefix + " invalid " + schemaName + " provided!"; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,169 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.common.data; |
|||
|
|||
import com.github.os72.protobuf.dynamic.DynamicSchema; |
|||
import com.google.protobuf.Descriptors; |
|||
import com.google.protobuf.DynamicMessage; |
|||
import com.squareup.wire.schema.internal.parser.ProtoFileElement; |
|||
import org.junit.Test; |
|||
import org.junit.runner.RunWith; |
|||
import org.mockito.junit.MockitoJUnitRunner; |
|||
|
|||
import java.util.List; |
|||
import java.util.Set; |
|||
|
|||
import static org.junit.Assert.assertEquals; |
|||
import static org.junit.Assert.assertNotNull; |
|||
import static org.junit.Assert.assertTrue; |
|||
|
|||
@RunWith(MockitoJUnitRunner.class) |
|||
public class DynamicProtoUtilsTest { |
|||
|
|||
@Test |
|||
public void testProtoSchemaWithMessageNestedTypes() throws Exception { |
|||
String schema = "syntax = \"proto3\";\n" + |
|||
"\n" + |
|||
"package testnested;\n" + |
|||
"\n" + |
|||
"message Outer {\n" + |
|||
" message MiddleAA {\n" + |
|||
" message Inner {\n" + |
|||
" optional int64 ival = 1;\n" + |
|||
" optional bool booly = 2;\n" + |
|||
" }\n" + |
|||
" Inner inner = 1;\n" + |
|||
" }\n" + |
|||
" message MiddleBB {\n" + |
|||
" message Inner {\n" + |
|||
" optional int32 ival = 1;\n" + |
|||
" optional bool booly = 2;\n" + |
|||
" }\n" + |
|||
" Inner inner = 1;\n" + |
|||
" }\n" + |
|||
" MiddleAA middleAA = 1;\n" + |
|||
" MiddleBB middleBB = 2;\n" + |
|||
"}"; |
|||
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(schema); |
|||
DynamicSchema dynamicSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, "test schema with nested types"); |
|||
assertNotNull(dynamicSchema); |
|||
Set<String> messageTypes = dynamicSchema.getMessageTypes(); |
|||
assertEquals(5, messageTypes.size()); |
|||
assertTrue(messageTypes.contains("testnested.Outer")); |
|||
assertTrue(messageTypes.contains("testnested.Outer.MiddleAA")); |
|||
assertTrue(messageTypes.contains("testnested.Outer.MiddleAA.Inner")); |
|||
assertTrue(messageTypes.contains("testnested.Outer.MiddleBB")); |
|||
assertTrue(messageTypes.contains("testnested.Outer.MiddleBB.Inner")); |
|||
|
|||
DynamicMessage.Builder middleAAInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); |
|||
Descriptors.Descriptor middleAAInnerMsgDescriptor = middleAAInnerMsgBuilder.getDescriptorForType(); |
|||
DynamicMessage middleAAInnerMsg = middleAAInnerMsgBuilder |
|||
.setField(middleAAInnerMsgDescriptor.findFieldByName("ival"), 1L) |
|||
.setField(middleAAInnerMsgDescriptor.findFieldByName("booly"), true) |
|||
.build(); |
|||
|
|||
DynamicMessage.Builder middleAAMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA"); |
|||
Descriptors.Descriptor middleAAMsgDescriptor = middleAAMsgBuilder.getDescriptorForType(); |
|||
DynamicMessage middleAAMsg = middleAAMsgBuilder |
|||
.setField(middleAAMsgDescriptor.findFieldByName("inner"), middleAAInnerMsg) |
|||
.build(); |
|||
|
|||
DynamicMessage.Builder middleBBInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner"); |
|||
Descriptors.Descriptor middleBBInnerMsgDescriptor = middleBBInnerMsgBuilder.getDescriptorForType(); |
|||
DynamicMessage middleBBInnerMsg = middleBBInnerMsgBuilder |
|||
.setField(middleBBInnerMsgDescriptor.findFieldByName("ival"), 0L) |
|||
.setField(middleBBInnerMsgDescriptor.findFieldByName("booly"), false) |
|||
.build(); |
|||
|
|||
DynamicMessage.Builder middleBBMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleBB"); |
|||
Descriptors.Descriptor middleBBMsgDescriptor = middleBBMsgBuilder.getDescriptorForType(); |
|||
DynamicMessage middleBBMsg = middleBBMsgBuilder |
|||
.setField(middleBBMsgDescriptor.findFieldByName("inner"), middleBBInnerMsg) |
|||
.build(); |
|||
|
|||
|
|||
DynamicMessage.Builder outerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer"); |
|||
Descriptors.Descriptor outerMsgBuilderDescriptor = outerMsgBuilder.getDescriptorForType(); |
|||
DynamicMessage outerMsg = outerMsgBuilder |
|||
.setField(outerMsgBuilderDescriptor.findFieldByName("middleAA"), middleAAMsg) |
|||
.setField(outerMsgBuilderDescriptor.findFieldByName("middleBB"), middleBBMsg) |
|||
.build(); |
|||
|
|||
assertEquals("{\n" + |
|||
" \"middleAA\": {\n" + |
|||
" \"inner\": {\n" + |
|||
" \"ival\": \"1\",\n" + |
|||
" \"booly\": true\n" + |
|||
" }\n" + |
|||
" },\n" + |
|||
" \"middleBB\": {\n" + |
|||
" \"inner\": {\n" + |
|||
" \"ival\": 0,\n" + |
|||
" \"booly\": false\n" + |
|||
" }\n" + |
|||
" }\n" + |
|||
"}", DynamicProtoUtils.dynamicMsgToJson(outerMsgBuilderDescriptor, outerMsg.toByteArray())); |
|||
} |
|||
|
|||
@Test |
|||
public void testProtoSchemaWithMessageOneOfs() throws Exception { |
|||
String schema = "syntax = \"proto3\";\n" + |
|||
"\n" + |
|||
"package testoneofs;\n" + |
|||
"\n" + |
|||
"message SubMessage {\n" + |
|||
" repeated string name = 1;\n" + |
|||
"}\n" + |
|||
"\n" + |
|||
"message SampleMessage {\n" + |
|||
" optional int32 id = 1;\n" + |
|||
" oneof testOneOf {\n" + |
|||
" string name = 4;\n" + |
|||
" SubMessage subMessage = 9;\n" + |
|||
" }\n" + |
|||
"}"; |
|||
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(schema); |
|||
DynamicSchema dynamicSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, "test schema with message oneOfs"); |
|||
assertNotNull(dynamicSchema); |
|||
Set<String> messageTypes = dynamicSchema.getMessageTypes(); |
|||
assertEquals(2, messageTypes.size()); |
|||
assertTrue(messageTypes.contains("testoneofs.SubMessage")); |
|||
assertTrue(messageTypes.contains("testoneofs.SampleMessage")); |
|||
|
|||
DynamicMessage.Builder sampleMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SampleMessage"); |
|||
Descriptors.Descriptor sampleMsgDescriptor = sampleMsgBuilder.getDescriptorForType(); |
|||
assertNotNull(sampleMsgDescriptor); |
|||
|
|||
List<Descriptors.FieldDescriptor> fields = sampleMsgDescriptor.getFields(); |
|||
assertEquals(3, fields.size()); |
|||
DynamicMessage sampleMsg = sampleMsgBuilder |
|||
.setField(sampleMsgDescriptor.findFieldByName("name"), "Bob") |
|||
.build(); |
|||
assertEquals("{\n" + " \"name\": \"Bob\"\n" + "}", DynamicProtoUtils.dynamicMsgToJson(sampleMsgDescriptor, sampleMsg.toByteArray())); |
|||
|
|||
DynamicMessage.Builder subMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SubMessage"); |
|||
Descriptors.Descriptor subMsgDescriptor = subMsgBuilder.getDescriptorForType(); |
|||
DynamicMessage subMsg = subMsgBuilder |
|||
.addRepeatedField(subMsgDescriptor.findFieldByName("name"), "Alice") |
|||
.addRepeatedField(subMsgDescriptor.findFieldByName("name"), "John") |
|||
.build(); |
|||
|
|||
DynamicMessage sampleMsgWithOneOfSubMessage = sampleMsgBuilder.setField(sampleMsgDescriptor.findFieldByName("subMessage"), subMsg).build(); |
|||
assertEquals("{\n" + " \"subMessage\": {\n" + " \"name\": [\"Alice\", \"John\"]\n" + " }\n" + "}", |
|||
DynamicProtoUtils.dynamicMsgToJson(sampleMsgDescriptor, sampleMsgWithOneOfSubMessage.toByteArray())); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,37 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.common.msg; |
|||
|
|||
import org.junit.jupiter.api.Test; |
|||
import org.thingsboard.server.common.data.FSTUtils; |
|||
import org.thingsboard.server.common.data.id.RuleChainId; |
|||
import org.thingsboard.server.common.data.id.RuleNodeId; |
|||
|
|||
import java.util.UUID; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
|
|||
class TbMsgProcessingStackItemTest { |
|||
|
|||
@Test |
|||
void testSerialization() { |
|||
TbMsgProcessingStackItem item = new TbMsgProcessingStackItem(new RuleChainId(UUID.randomUUID()), new RuleNodeId(UUID.randomUUID())); |
|||
byte[] bytes = FSTUtils.encode(item); |
|||
TbMsgProcessingStackItem itemDecoded = FSTUtils.decode(bytes); |
|||
assertThat(item).isEqualTo(itemDecoded); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,35 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.common.util; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.fasterxml.jackson.databind.node.ObjectNode; |
|||
import org.junit.Assert; |
|||
import org.junit.Test; |
|||
|
|||
public class JacksonUtilTest { |
|||
|
|||
@Test |
|||
public void allow_unquoted_field_mapper_test() { |
|||
String data = "{data: 123}"; |
|||
JsonNode actualResult = JacksonUtil.toJsonNode(data, JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER); // should be: {"data": 123}
|
|||
ObjectNode expectedResult = JacksonUtil.newObjectNode(); |
|||
expectedResult.put("data", 123); // {"data": 123}
|
|||
Assert.assertEquals(expectedResult, actualResult); |
|||
Assert.assertThrows(IllegalArgumentException.class, () -> JacksonUtil.toJsonNode(data)); // syntax exception due to missing quotes in the field name!
|
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,338 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.rule.engine.metadata; |
|||
|
|||
import com.datastax.oss.driver.api.core.uuid.Uuids; |
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.fasterxml.jackson.databind.ObjectMapper; |
|||
import com.fasterxml.jackson.databind.node.ObjectNode; |
|||
import com.google.common.util.concurrent.Futures; |
|||
import org.junit.After; |
|||
import org.junit.Assert; |
|||
import org.junit.Before; |
|||
import org.junit.Test; |
|||
import org.junit.runner.RunWith; |
|||
import org.mockito.ArgumentCaptor; |
|||
import org.mockito.Mock; |
|||
import org.mockito.Mockito; |
|||
import org.mockito.junit.MockitoJUnitRunner; |
|||
import org.thingsboard.common.util.AbstractListeningExecutor; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.rule.engine.api.TbContext; |
|||
import org.thingsboard.rule.engine.api.TbNodeConfiguration; |
|||
import org.thingsboard.rule.engine.api.TbNodeException; |
|||
import org.thingsboard.server.common.data.DataConstants; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BasicTsKvEntry; |
|||
import org.thingsboard.server.common.data.kv.JsonDataEntry; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.msg.TbMsg; |
|||
import org.thingsboard.server.common.msg.TbMsgMetaData; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.timeseries.TimeseriesService; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
import java.util.stream.Collectors; |
|||
|
|||
import static org.mockito.ArgumentMatchers.any; |
|||
import static org.mockito.Mockito.lenient; |
|||
import static org.mockito.Mockito.never; |
|||
|
|||
@RunWith(MockitoJUnitRunner.class) |
|||
public class TbAbstractGetAttributesNodeTest { |
|||
|
|||
final ObjectMapper mapper = new ObjectMapper(); |
|||
|
|||
private EntityId originator = new DeviceId(Uuids.timeBased()); |
|||
private TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); |
|||
|
|||
@Mock |
|||
private TbContext ctx; |
|||
@Mock |
|||
private AttributesService attributesService; |
|||
@Mock |
|||
private TimeseriesService tsService; |
|||
private AbstractListeningExecutor dbExecutor; |
|||
|
|||
private List<String> clientAttributes; |
|||
private List<String> serverAttributes; |
|||
private List<String> sharedAttributes; |
|||
private List<String> tsKeys; |
|||
private long ts; |
|||
|
|||
@Before |
|||
public void before() throws TbNodeException { |
|||
dbExecutor = new AbstractListeningExecutor() { |
|||
@Override |
|||
protected int getThreadPollSize() { |
|||
return 3; |
|||
} |
|||
}; |
|||
dbExecutor.init(); |
|||
|
|||
Mockito.reset(ctx); |
|||
Mockito.reset(attributesService); |
|||
Mockito.reset(tsService); |
|||
|
|||
Mockito.reset(ctx); |
|||
Mockito.reset(attributesService); |
|||
Mockito.reset(tsService); |
|||
|
|||
lenient().when(ctx.getAttributesService()).thenReturn(attributesService); |
|||
lenient().when(ctx.getTimeseriesService()).thenReturn(tsService); |
|||
lenient().when(ctx.getTenantId()).thenReturn(tenantId); |
|||
lenient().when(ctx.getDbCallbackExecutor()).thenReturn(dbExecutor); |
|||
|
|||
clientAttributes = getAttributeNames("client"); |
|||
serverAttributes = getAttributeNames("server"); |
|||
sharedAttributes = getAttributeNames("shared"); |
|||
tsKeys = List.of("temperature", "humidity", "unknown"); |
|||
ts = System.currentTimeMillis(); |
|||
|
|||
Mockito.when(attributesService.find(tenantId, originator, DataConstants.CLIENT_SCOPE, clientAttributes)) |
|||
.thenReturn(Futures.immediateFuture(getListAttributeKvEntry(clientAttributes, ts))); |
|||
|
|||
|
|||
Mockito.when(attributesService.find(tenantId, originator, DataConstants.SERVER_SCOPE, serverAttributes)) |
|||
.thenReturn(Futures.immediateFuture(getListAttributeKvEntry(serverAttributes, ts))); |
|||
|
|||
|
|||
Mockito.when(attributesService.find(tenantId, originator, DataConstants.SHARED_SCOPE, sharedAttributes)) |
|||
.thenReturn(Futures.immediateFuture(getListAttributeKvEntry(sharedAttributes, ts))); |
|||
|
|||
Mockito.when(tsService.findLatest(tenantId, originator, tsKeys)) |
|||
.thenReturn(Futures.immediateFuture(getListTsKvEntry(tsKeys, ts))); |
|||
} |
|||
|
|||
@After |
|||
public void after() { |
|||
dbExecutor.destroy(); |
|||
} |
|||
|
|||
@Test |
|||
public void fetchToMetadata_whenOnMsg_then_success() throws Exception { |
|||
TbGetAttributesNode node = initNode(false, false, false); |
|||
TbMsg msg = getTbMsg(originator); |
|||
node.onMsg(ctx, msg); |
|||
|
|||
TbMsg resultMsg = checkMsg(); |
|||
TbMsgMetaData msgMetaData = resultMsg.getMetaData(); |
|||
|
|||
//check attributes
|
|||
checkAttributes(clientAttributes, "cs_", false, msgMetaData, null); |
|||
checkAttributes(serverAttributes, "ss_", false, msgMetaData, null); |
|||
checkAttributes(sharedAttributes, "shared_", false, msgMetaData, null); |
|||
|
|||
//check timeseries
|
|||
checkTs(tsKeys, false, false, msgMetaData, null); |
|||
} |
|||
|
|||
@Test |
|||
public void fetchToMetadata_latestWithTs_whenOnMsg_then_success() throws Exception { |
|||
TbGetAttributesNode node = initNode(false, true, false); |
|||
TbMsg msg = getTbMsg(originator); |
|||
node.onMsg(ctx, msg); |
|||
|
|||
TbMsg resultMsg = checkMsg(); |
|||
TbMsgMetaData msgMetaData = resultMsg.getMetaData(); |
|||
|
|||
//check attributes
|
|||
checkAttributes(clientAttributes, "cs_", false, msgMetaData, null); |
|||
checkAttributes(serverAttributes, "ss_", false, msgMetaData, null); |
|||
checkAttributes(sharedAttributes, "shared_", false, msgMetaData, null); |
|||
|
|||
//check timeseries with ts
|
|||
checkTs(tsKeys, false, true, msgMetaData, null); |
|||
} |
|||
|
|||
@Test |
|||
public void fetchToData_whenOnMsg_then_success() throws Exception { |
|||
TbGetAttributesNode node = initNode(true, false, false); |
|||
TbMsg msg = getTbMsg(originator); |
|||
node.onMsg(ctx, msg); |
|||
|
|||
TbMsg resultMsg = checkMsg(); |
|||
JsonNode msgData = JacksonUtil.toJsonNode(resultMsg.getData()); |
|||
|
|||
//check attributes
|
|||
checkAttributes(clientAttributes, "cs_", true, null, msgData); |
|||
checkAttributes(serverAttributes, "ss_", true, null, msgData); |
|||
checkAttributes(sharedAttributes, "shared_", true, null, msgData); |
|||
|
|||
//check timeseries
|
|||
checkTs(tsKeys, true, false, null, msgData); |
|||
} |
|||
|
|||
@Test |
|||
public void fetchToData_latestWithTs_whenOnMsg_then_success() throws Exception { |
|||
TbGetAttributesNode node = initNode(true, true, false); |
|||
TbMsg msg = getTbMsg(originator); |
|||
node.onMsg(ctx, msg); |
|||
|
|||
TbMsg resultMsg = checkMsg(); |
|||
JsonNode msgData = JacksonUtil.toJsonNode(resultMsg.getData()); |
|||
|
|||
//check attributes
|
|||
checkAttributes(clientAttributes, "cs_", true, null, msgData); |
|||
checkAttributes(serverAttributes, "ss_", true, null, msgData); |
|||
checkAttributes(sharedAttributes, "shared_", true, null, msgData); |
|||
|
|||
//check timeseries with ts
|
|||
checkTs(tsKeys, true, true, null, msgData); |
|||
} |
|||
|
|||
@Test |
|||
public void fetchToData_whenOnMsg_then_failure() throws Exception { |
|||
TbGetAttributesNode node = initNode(true, true, true); |
|||
TbMsg msg = getTbMsg(originator); |
|||
node.onMsg(ctx, msg); |
|||
|
|||
ArgumentCaptor<TbMsg> newMsgCaptor = ArgumentCaptor.forClass(TbMsg.class); |
|||
ArgumentCaptor<Exception> exceptionCaptor = ArgumentCaptor.forClass(Exception.class); |
|||
Mockito.verify(ctx, never()).tellSuccess(any()); |
|||
Mockito.verify(ctx, Mockito.timeout(5000)).tellFailure(newMsgCaptor.capture(), exceptionCaptor.capture()); |
|||
|
|||
Assert.assertSame(newMsgCaptor.getValue(), msg); |
|||
Assert.assertNotNull(exceptionCaptor.getValue()); |
|||
} |
|||
|
|||
@Test |
|||
public void fetchToData_whenOnMsg_then_data_not_object_failure() throws Exception { |
|||
TbGetAttributesNode node = initNode(true, true, true); |
|||
TbMsg msg = TbMsg.newMsg("TEST", originator, new TbMsgMetaData(), "[]"); |
|||
node.onMsg(ctx, msg); |
|||
|
|||
ArgumentCaptor<TbMsg> newMsgCaptor = ArgumentCaptor.forClass(TbMsg.class); |
|||
ArgumentCaptor<Exception> exceptionCaptor = ArgumentCaptor.forClass(Exception.class); |
|||
Mockito.verify(ctx, never()).tellSuccess(any()); |
|||
Mockito.verify(ctx, Mockito.timeout(5000)).tellFailure(newMsgCaptor.capture(), exceptionCaptor.capture()); |
|||
|
|||
Assert.assertSame(newMsgCaptor.getValue(), msg); |
|||
Assert.assertNotNull(exceptionCaptor.getValue()); |
|||
} |
|||
|
|||
private TbMsg checkMsg() { |
|||
ArgumentCaptor<TbMsg> msgCaptor = ArgumentCaptor.forClass(TbMsg.class); |
|||
Mockito.verify(ctx, Mockito.timeout(5000)).tellSuccess(msgCaptor.capture()); |
|||
|
|||
TbMsg resultMsg = msgCaptor.getValue(); |
|||
Assert.assertNotNull(resultMsg); |
|||
Assert.assertNotNull(resultMsg.getMetaData()); |
|||
Assert.assertNotNull(resultMsg.getData()); |
|||
return resultMsg; |
|||
} |
|||
|
|||
private void checkAttributes(List<String> attributes, String prefix, boolean fetchToData, TbMsgMetaData msgMetaData, JsonNode msgData) { |
|||
attributes.stream() |
|||
.filter(attribute -> !attribute.equals("unknown")) |
|||
.forEach(attribute -> { |
|||
String result; |
|||
if (fetchToData) { |
|||
result = msgData.get(prefix + attribute).asText(); |
|||
} else { |
|||
result = msgMetaData.getValue(prefix + attribute); |
|||
} |
|||
Assert.assertNotNull(result); |
|||
Assert.assertEquals(attribute + "_value", result); |
|||
}); |
|||
} |
|||
|
|||
private void checkTs(List<String> tsKeys, boolean fetchToData, boolean getLatestValueWithTs, TbMsgMetaData msgMetaData, JsonNode msgData) { |
|||
long value = 1L; |
|||
for (String key : tsKeys) { |
|||
if (key.equals("unknown")) { |
|||
continue; |
|||
} |
|||
String actualValue; |
|||
String expectedValue; |
|||
if (getLatestValueWithTs) { |
|||
expectedValue = "{\"ts\":" + ts + ",\"value\":{\"data\":" + value + "}}"; |
|||
} else { |
|||
expectedValue = "{\"data\":" + value + "}"; |
|||
} |
|||
if (fetchToData) { |
|||
actualValue = JacksonUtil.toString(msgData.get(key)); |
|||
} else { |
|||
actualValue = msgMetaData.getValue(key); |
|||
} |
|||
Assert.assertNotNull(actualValue); |
|||
Assert.assertEquals(expectedValue, actualValue); |
|||
value++; |
|||
} |
|||
} |
|||
|
|||
private TbGetAttributesNode initNode(boolean fetchToData, boolean getLatestValueWithTs, boolean isTellFailureIfAbsent) throws TbNodeException { |
|||
TbGetAttributesNodeConfiguration config = new TbGetAttributesNodeConfiguration(); |
|||
config.setClientAttributeNames(List.of("client_attr_1", "client_attr_2", "${client_attr_metadata}", "unknown")); |
|||
config.setServerAttributeNames(List.of("server_attr_1", "server_attr_2", "${server_attr_metadata}", "unknown")); |
|||
config.setSharedAttributeNames(List.of("shared_attr_1", "shared_attr_2", "$[shared_attr_data]", "unknown")); |
|||
config.setLatestTsKeyNames(List.of("temperature", "humidity", "unknown")); |
|||
config.setFetchToData(fetchToData); |
|||
config.setGetLatestValueWithTs(getLatestValueWithTs); |
|||
config.setTellFailureIfAbsent(isTellFailureIfAbsent); |
|||
TbNodeConfiguration nodeConfiguration = new TbNodeConfiguration(mapper.valueToTree(config)); |
|||
TbGetAttributesNode node = new TbGetAttributesNode(); |
|||
node.init(ctx, nodeConfiguration); |
|||
return node; |
|||
} |
|||
|
|||
private TbMsg getTbMsg(EntityId entityId) { |
|||
ObjectNode msgData = JacksonUtil.newObjectNode(); |
|||
msgData.put("shared_attr_data", "shared_attr_3"); |
|||
|
|||
TbMsgMetaData msgMetaData = new TbMsgMetaData(); |
|||
msgMetaData.putValue("client_attr_metadata", "client_attr_3"); |
|||
msgMetaData.putValue("server_attr_metadata", "server_attr_3"); |
|||
|
|||
return TbMsg.newMsg("TEST", entityId, msgMetaData, msgData.toString()); |
|||
} |
|||
|
|||
private List<String> getAttributeNames(String prefix) { |
|||
return List.of(prefix + "_attr_1", prefix + "_attr_2", prefix + "_attr_3", "unknown"); |
|||
} |
|||
|
|||
private List<AttributeKvEntry> getListAttributeKvEntry(List<String> attributes, long ts) { |
|||
return attributes.stream() |
|||
.filter(attribute -> !attribute.equals("unknown")) |
|||
.map(attribute -> toAttributeKvEntry(ts, attribute)) |
|||
.collect(Collectors.toList()); |
|||
} |
|||
|
|||
private BaseAttributeKvEntry toAttributeKvEntry(long ts, String attribute) { |
|||
return new BaseAttributeKvEntry(ts, new StringDataEntry(attribute, attribute + "_value")); |
|||
} |
|||
|
|||
private List<TsKvEntry> getListTsKvEntry(List<String> keys, long ts) { |
|||
long value = 1L; |
|||
List<TsKvEntry> kvEntries = new ArrayList<>(); |
|||
for (String key : keys) { |
|||
if (key.equals("unknown")) { |
|||
continue; |
|||
} |
|||
String dataValue = "{\"data\":" + value + "}"; |
|||
kvEntries.add(new BasicTsKvEntry(ts, new JsonDataEntry(key, dataValue))); |
|||
value++; |
|||
} |
|||
return kvEntries; |
|||
} |
|||
|
|||
} |
|||
Loading…
Reference in new issue