added extra properties to kafka consumer (#2751)

* added extra properties to kafka consumer

* added default values for kafka consumer properties in TbKafkaSettings

* Update TbKafkaSettings.java

* Update thingsboard.yml

Co-authored-by: Andrew Shvayka <ashvayka@thingsboard.io>

application/src/main/resources/thingsboard.yml

@@ -584,6 +584,9 @@ queue:
     linger.ms: "${TB_KAFKA_LINGER_MS:1}"
     buffer.memory: "${TB_BUFFER_MEMORY:33554432}"
     replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}"
+    max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}"
+    max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}"
+    fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}"
     topic-properties:
       rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
       core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"

common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java

@@ -54,6 +54,9 @@ public class TbKafkaConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQue
         if (groupId != null) {
             props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
         }
+        props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, settings.getMaxPollRecords());
+        props.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, settings.getMaxPartitionFetchBytes());
+        props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, settings.getFetchMaxBytes());
         props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit);
         props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs);
         props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");

common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java

@@ -55,6 +55,18 @@ public class TbKafkaSettings {
     @Getter
     private short replicationFactor;
 
+    @Value("${queue.kafka.max_poll_records:8192}")
+    @Getter
+    private int maxPollRecords;
+
+    @Value("${queue.kafka.max_partition_fetch_bytes:16777216}")
+    @Getter
+    private int maxPartitionFetchBytes;
+
+    @Value("${queue.kafka.fetch_max_bytes:134217728}")
+    @Getter
+    private  int fetchMaxBytes;
+
     @Value("${kafka.other:#{null}}")
     private List<TbKafkaProperty> other;