tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Address review feedback: limit preservingProtoFieldNames to dynamic proto only and add config to all transport ymls 

Co-authored-by: ViacheslavKlimov <56742475+ViacheslavKlimov@users.noreply.github.com>
pull/14725/head
copilot-swe-agent[bot] 7 months ago
parent
4b38a7784e
commit
cc57bd3fd2
9 changed files with 97 additions and 29 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 77
      common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java
  2. 7
      common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java
  3. 8
      common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java
  4. 9
      common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/util/LwM2MClientSerDes.java
  5. 5
      transport/coap/src/main/resources/tb-coap-transport.yml
  6. 5
      transport/http/src/main/resources/tb-http-transport.yml
  7. 5
      transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml
  8. 5
      transport/mqtt/src/main/resources/tb-mqtt-transport.yml
  9. 5
      transport/snmp/src/main/resources/tb-snmp-transport.yml

77
common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java
View File

@ -169,8 +169,8 @@ public class DynamicProtoUtilsTest {
@Test
public void testProtoSchemaDefaultBehaviorConvertsToCamelCase() throws Exception {
// By default (when TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES is not set),
// field names should be converted to camelCase for backward compatibility
// Test default behavior when TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES is not set
// Field names should be converted to camelCase for backward compatibility
String schema = "syntax = \"proto3\";\n" +
"\n" +
"package firmware;\n" +
@ -200,7 +200,63 @@ public class DynamicProtoUtilsTest {
String json = DynamicProtoUtils.dynamicMsgToJson(firmwareStatusDescriptor, firmwareStatus.toByteArray());
// By default, field names should be converted to camelCase (backward compatible behavior)
// Check the actual behavior based on the current flag setting
boolean preserveFieldNames = Boolean.parseBoolean(System.getProperty("transport.json.preserve_proto_field_names", System.getenv("TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES")));
if (!preserveFieldNames) {
// Default behavior: field names converted to camelCase
assertTrue("JSON should contain camelCase field 'currentFwTitle'", json.contains("\"currentFwTitle\""));
assertTrue("JSON should contain camelCase field 'currentFwVersion'", json.contains("\"currentFwVersion\""));
assertTrue("JSON should contain camelCase field 'fwState'", json.contains("\"fwState\""));
assertTrue("JSON should contain camelCase field 'targetFwTitle'", json.contains("\"targetFwTitle\""));
assertTrue("JSON should contain camelCase field 'targetFwVersion'", json.contains("\"targetFwVersion\""));
// Verify snake_case versions are NOT present
assertFalse("JSON should NOT contain snake_case field 'current_fw_title'", json.contains("\"current_fw_title\""));
assertFalse("JSON should NOT contain snake_case field 'fw_state'", json.contains("\"fw_state\""));
} else {
// This test is designed to verify default behavior, skip if flag is set
// The next test will verify the preserve behavior
assertTrue("This test expects default behavior (camelCase conversion). " +
"Set TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES=false or run without the flag.",
!preserveFieldNames);
}
}
@Test
public void testProtoSchemaPreservesSnakeCaseFieldNamesWhenEnabled() throws Exception {
// Test behavior when TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES is set to true
// Field names should be preserved as defined in .proto schema
String schema = "syntax = \"proto3\";\n" +
"\n" +
"package firmware;\n" +
"\n" +
"message FirmwareStatus {\n" +
" string current_fw_title = 1;\n" +
" string current_fw_version = 2;\n" +
" string fw_state = 3;\n" +
" string target_fw_title = 4;\n" +
" string target_fw_version = 5;\n" +
"}";
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(schema);
DynamicSchema dynamicSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, "test schema with snake_case fields");
assertNotNull(dynamicSchema);
DynamicMessage.Builder firmwareStatusBuilder = dynamicSchema.newMessageBuilder("firmware.FirmwareStatus");
Descriptors.Descriptor firmwareStatusDescriptor = firmwareStatusBuilder.getDescriptorForType();
assertNotNull(firmwareStatusDescriptor);
DynamicMessage firmwareStatus = firmwareStatusBuilder
.setField(firmwareStatusDescriptor.findFieldByName("current_fw_title"), "firmware_v1")
.setField(firmwareStatusDescriptor.findFieldByName("current_fw_version"), "1.0.0")
.setField(firmwareStatusDescriptor.findFieldByName("fw_state"), "DOWNLOADING")
.setField(firmwareStatusDescriptor.findFieldByName("target_fw_title"), "firmware_v2")
.setField(firmwareStatusDescriptor.findFieldByName("target_fw_version"), "2.0.0")
.build();
String json = DynamicProtoUtils.dynamicMsgToJson(firmwareStatusDescriptor, firmwareStatus.toByteArray());
// Check the actual behavior based on the current flag setting
boolean preserveFieldNames = Boolean.parseBoolean(System.getProperty("transport.json.preserve_proto_field_names", System.getenv("TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES")));
if (preserveFieldNames) {
@ -210,13 +266,16 @@ public class DynamicProtoUtilsTest {
assertTrue("JSON should contain snake_case field 'fw_state'", json.contains("\"fw_state\""));
assertTrue("JSON should contain snake_case field 'target_fw_title'", json.contains("\"target_fw_title\""));
assertTrue("JSON should contain snake_case field 'target_fw_version'", json.contains("\"target_fw_version\""));
// Verify camelCase versions are NOT present
assertFalse("JSON should NOT contain camelCase field 'currentFwTitle'", json.contains("\"currentFwTitle\""));
assertFalse("JSON should NOT contain camelCase field 'fwState'", json.contains("\"fwState\""));
} else {
// Default behavior: field names converted to camelCase
assertTrue("JSON should contain camelCase field 'currentFwTitle'", json.contains("\"currentFwTitle\""));
assertTrue("JSON should contain camelCase field 'currentFwVersion'", json.contains("\"currentFwVersion\""));
assertTrue("JSON should contain camelCase field 'fwState'", json.contains("\"fwState\""));
assertTrue("JSON should contain camelCase field 'targetFwTitle'", json.contains("\"targetFwTitle\""));
assertTrue("JSON should contain camelCase field 'targetFwVersion'", json.contains("\"targetFwVersion\""));
// This test is designed to verify preserve behavior, skip if flag is not set
// Run this test with: TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES=true
assertTrue("This test expects preserve behavior (snake_case preservation). " +
"Set TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES=true to run this test.",
preserveFieldNames);
}
}

7
common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java
View File

@ -24,10 +24,6 @@ import java.util.UUID;
public class TbProtoJsQueueMsg<T extends com.google.protobuf.GeneratedMessageV3> extends TbProtoQueueMsg<T> {
private static final JsonFormat.Printer JSON_PRINTER = JsonFormat.printer();
private static final JsonFormat.Printer JSON_PRINTER_PRESERVING_PROTO_FIELD_NAMES = JsonFormat.printer().preservingProtoFieldNames();
private static final boolean PRESERVE_PROTO_FIELD_NAMES = Boolean.parseBoolean(System.getProperty("transport.json.preserve_proto_field_names", System.getenv("TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES")));
public TbProtoJsQueueMsg(UUID key, T value) {
super(key, value);
}
@ -39,8 +35,7 @@ public class TbProtoJsQueueMsg<T extends com.google.protobuf.GeneratedMessageV3>
@Override
public byte[] getData() {
try {
JsonFormat.Printer printer = PRESERVE_PROTO_FIELD_NAMES ? JSON_PRINTER_PRESERVING_PROTO_FIELD_NAMES : JSON_PRINTER;
return printer.print(value).getBytes(StandardCharsets.UTF_8);
return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8);
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}

8
common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java
View File

@ -27,16 +27,10 @@ import java.nio.charset.StandardCharsets;
* Created by ashvayka on 25.09.18.
*/
public class RemoteJsRequestEncoder implements TbKafkaEncoder<TbProtoQueueMsg<JsInvokeProtos.RemoteJsRequest>> {
private static final JsonFormat.Printer JSON_PRINTER = JsonFormat.printer();
private static final JsonFormat.Printer JSON_PRINTER_PRESERVING_PROTO_FIELD_NAMES = JsonFormat.printer().preservingProtoFieldNames();
private static final boolean PRESERVE_PROTO_FIELD_NAMES = Boolean.parseBoolean(System.getProperty("transport.json.preserve_proto_field_names", System.getenv("TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES")));
@Override
public byte[] encode(TbProtoQueueMsg<JsInvokeProtos.RemoteJsRequest> value) {
try {
JsonFormat.Printer printer = PRESERVE_PROTO_FIELD_NAMES ? JSON_PRINTER_PRESERVING_PROTO_FIELD_NAMES : JSON_PRINTER;
return printer.print(value.getValue()).getBytes(StandardCharsets.UTF_8);
return JsonFormat.printer().print(value.getValue()).getBytes(StandardCharsets.UTF_8);
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}

9
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/util/LwM2MClientSerDes.java
View File

@ -53,9 +53,6 @@ import static org.thingsboard.common.util.JacksonUtil.toJsonNode;
public class LwM2MClientSerDes {
public static final String VALUE = "value";
private static final RegistrationSerDes registrationSerDes = new RegistrationSerDes();
private static final JsonFormat.Printer JSON_PRINTER = JsonFormat.printer();
private static final JsonFormat.Printer JSON_PRINTER_PRESERVING_PROTO_FIELD_NAMES = JsonFormat.printer().preservingProtoFieldNames();
private static final boolean PRESERVE_PROTO_FIELD_NAMES = Boolean.parseBoolean(System.getProperty("transport.json.preserve_proto_field_names", System.getenv("TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES")));
@SneakyThrows
public static byte[] serialize(LwM2mClient client) {
@ -74,8 +71,7 @@ public class LwM2MClientSerDes {
JsonObject sharedAttributes = new JsonObject();
for (Map.Entry<String, TransportProtos.TsKvProto> entry : client.getSharedAttributes().entrySet()) {
JsonFormat.Printer printer = PRESERVE_PROTO_FIELD_NAMES ? JSON_PRINTER_PRESERVING_PROTO_FIELD_NAMES : JSON_PRINTER;
sharedAttributes.addProperty(entry.getKey(), printer.print(entry.getValue()));
sharedAttributes.addProperty(entry.getKey(), JsonFormat.printer().print(entry.getValue()));
}
o.add("sharedAttributes", sharedAttributes);
@ -88,8 +84,7 @@ public class LwM2MClientSerDes {
o.addProperty("state", client.getState().toString());
if (client.getSession() != null) {
JsonFormat.Printer printer = PRESERVE_PROTO_FIELD_NAMES ? JSON_PRINTER_PRESERVING_PROTO_FIELD_NAMES : JSON_PRINTER;
o.addProperty("session", printer.print(client.getSession()));
o.addProperty("session", JsonFormat.printer().print(client.getSession()));
}
if (client.getTenantId() != null) {
o.addProperty("tenantId", client.getTenantId().toString());

5
transport/coap/src/main/resources/tb-coap-transport.yml
View File

@ -158,6 +158,11 @@ transport:
type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}"
# Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check)
max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}"
# Preserve proto field names (e.g., 'current_fw_title') instead of converting to camelCase (e.g., 'currentFwTitle') when processing Protobuf messages.
# When set to 'false' (default), field names are converted to camelCase for backward compatibility.
# When set to 'true', field names are preserved as defined in the .proto schema.
# This affects dynamic Protobuf messages used in device communication.
preserve_proto_field_names: "${TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES:false}"
log:
# Enable/Disable log of transport messages to telemetry. For example, logging of LwM2M registration update
enabled: "${TB_TRANSPORT_LOG_ENABLED:true}"

5
transport/http/src/main/resources/tb-http-transport.yml
View File

@ -189,6 +189,11 @@ transport:
type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}"
# Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check)
max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}"
# Preserve proto field names (e.g., 'current_fw_title') instead of converting to camelCase (e.g., 'currentFwTitle') when processing Protobuf messages.
# When set to 'false' (default), field names are converted to camelCase for backward compatibility.
# When set to 'true', field names are preserved as defined in the .proto schema.
# This affects dynamic Protobuf messages used in device communication.
preserve_proto_field_names: "${TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES:false}"
log:
# Enable/Disable log of transport messages to telemetry. For example, logging of LwM2M registration update
enabled: "${TB_TRANSPORT_LOG_ENABLED:true}"

5
transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml
View File

@ -149,6 +149,11 @@ transport:
type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:false}"
# Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check)
max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}"
# Preserve proto field names (e.g., 'current_fw_title') instead of converting to camelCase (e.g., 'currentFwTitle') when processing Protobuf messages.
# When set to 'false' (default), field names are converted to camelCase for backward compatibility.
# When set to 'true', field names are preserved as defined in the .proto schema.
# This affects dynamic Protobuf messages used in device communication.
preserve_proto_field_names: "${TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES:false}"
client_side_rpc:
# Processing timeout interval of the RPC command on the CLIENT SIDE. Time in milliseconds
timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}"

5
transport/mqtt/src/main/resources/tb-mqtt-transport.yml
View File

@ -212,6 +212,11 @@ transport:
type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}"
# Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check)
max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}"
# Preserve proto field names (e.g., 'current_fw_title') instead of converting to camelCase (e.g., 'currentFwTitle') when processing Protobuf messages.
# When set to 'false' (default), field names are converted to camelCase for backward compatibility.
# When set to 'true', field names are preserved as defined in the .proto schema.
# This affects dynamic Protobuf messages used in device communication.
preserve_proto_field_names: "${TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES:false}"
log:
# Enable/Disable log of transport messages to telemetry. For example, logging of LwM2M registration update
enabled: "${TB_TRANSPORT_LOG_ENABLED:true}"

5
transport/snmp/src/main/resources/tb-snmp-transport.yml
View File

@ -170,6 +170,11 @@ transport:
type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}"
# Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check)
max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}"
# Preserve proto field names (e.g., 'current_fw_title') instead of converting to camelCase (e.g., 'currentFwTitle') when processing Protobuf messages.
# When set to 'false' (default), field names are converted to camelCase for backward compatibility.
# When set to 'true', field names are preserved as defined in the .proto schema.
# This affects dynamic Protobuf messages used in device communication.
preserve_proto_field_names: "${TB_TRANSPORT_JSON_PRESERVE_PROTO_FIELD_NAMES:false}"
log:
# Enable/Disable log of transport messages to telemetry. For example, logging of LwM2M registration update
enabled: "${TB_TRANSPORT_LOG_ENABLED:true}"

Write Preview
Loading…
Cancel
Save