From 18ca330c9ea4120c6d8d28d18ad0d1adf13c89e9 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 28 Aug 2020 17:02:04 +0300 Subject: [PATCH 1/5] in memory queue logs --- application/src/main/resources/logback.xml | 2 ++ application/src/main/resources/thingsboard.yml | 4 ++++ .../server/queue/memory/InMemoryStorage.java | 15 ++------------- .../queue/memory/InMemoryTbQueueProducer.java | 2 +- .../provider/InMemoryMonolithQueueFactory.java | 7 +++++++ 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 01aec9ceb9..81eb788e35 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -29,6 +29,8 @@ + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index b6894bb0e7..ac76b67cd3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -594,6 +594,10 @@ swagger: queue: type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + in_memory: + stats: + # For debug lvl + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index 994ca26305..31f7958bbf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -23,27 +23,21 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; @Slf4j public final class InMemoryStorage { private static InMemoryStorage instance; private final ConcurrentHashMap> storage; - private static ScheduledExecutorService statExecutor; private InMemoryStorage() { storage = new ConcurrentHashMap<>(); - statExecutor = Executors.newSingleThreadScheduledExecutor(); - statExecutor.scheduleAtFixedRate(this::printStats, 60, 60, TimeUnit.SECONDS); } - private void printStats() { + public void printStats() { storage.forEach((topic, queue) -> { if (queue.size() > 0) { - log.debug("Topic: [{}], Queue size: [{}]", topic, queue.size()); + log.debug("[{}] Queue Size [{}]", topic, queue.size()); } }); } @@ -90,9 +84,4 @@ public final class InMemoryStorage { storage.clear(); } - public void destroy() { - if (statExecutor != null) { - statExecutor.shutdownNow(); - } - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index 84a9a1fdf0..cfcd788a16 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -53,6 +53,6 @@ public class InMemoryTbQueueProducer implements TbQueuePro @Override public void stop() { - storage.destroy(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index fbdbeaab0c..8e2ff01e28 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -28,6 +29,7 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -120,4 +122,9 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") + private void printInMemoryStats() { + InMemoryStorage.getInstance().printStats(); + } } From e5ad5e0b301b7434def69817732f84b33d018040 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 28 Aug 2020 17:36:57 +0300 Subject: [PATCH 2/5] refactored --- .../server/queue/provider/InMemoryMonolithQueueFactory.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 8e2ff01e28..8ebda6e7b9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -49,6 +49,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final InMemoryStorage storage; public InMemoryMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -61,6 +62,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; + this.storage = InMemoryStorage.getInstance(); } @Override @@ -125,6 +127,6 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { - InMemoryStorage.getInstance().printStats(); + storage.printStats(); } } From bc2520fbbb95d06e63dea73f5c5d47fc08ce5aee Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 1 Sep 2020 14:53:47 +0300 Subject: [PATCH 3/5] add request param "orderBy" for getTimeseries --- .../thingsboard/server/controller/TelemetryController.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index bb866e3e9b..af4eb0f08f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -197,19 +197,21 @@ public class TelemetryController extends BaseController { @RequestMapping(value = "/{entityType}/{entityId}/values/timeseries", method = RequestMethod.GET, params = {"keys", "startTs", "endTs"}) @ResponseBody public DeferredResult getTimeseries( - @PathVariable("entityType") String entityType, @PathVariable("entityId") String entityIdStr, + @PathVariable("entityType") String entityType, + @PathVariable("entityId") String entityIdStr, @RequestParam(name = "keys") String keys, @RequestParam(name = "startTs") Long startTs, @RequestParam(name = "endTs") Long endTs, @RequestParam(name = "interval", defaultValue = "0") Long interval, @RequestParam(name = "limit", defaultValue = "100") Integer limit, @RequestParam(name = "agg", defaultValue = "NONE") String aggStr, + @RequestParam(name= "orderBy", defaultValue = "DESC") String orderBy, @RequestParam(name = "useStrictDataTypes", required = false, defaultValue = "false") Boolean useStrictDataTypes) throws ThingsboardException { return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr, (result, tenantId, entityId) -> { // If interval is 0, convert this to a NONE aggregation, which is probably what the user really wanted Aggregation agg = interval == 0L ? Aggregation.valueOf(Aggregation.NONE.name()) : Aggregation.valueOf(aggStr); - List queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, interval, limit, agg)) + List queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, interval, limit, agg, orderBy)) .collect(Collectors.toList()); Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor()); From 34dbc81de6890a37a28043377f0a10d3ffab54de Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Tue, 1 Sep 2020 15:47:06 +0300 Subject: [PATCH 4/5] fix drop partitions by max ttl procedure --- .../schema_update_psql_drop_partitions.sql | 1 + .../install/ThingsboardInstallService.java | 5 +++++ .../CassandraTsDatabaseUpgradeService.java | 1 + .../install/PsqlTsDatabaseUpgradeService.java | 6 ++++++ .../TimescaleTsDatabaseUpgradeService.java | 2 ++ .../service/ttl/AbstractCleanUpService.java | 18 +++++++----------- .../ttl/events/EventsCleanUpService.java | 2 +- .../PsqlTimeseriesCleanUpService.java | 3 ++- .../TimescaleTimeseriesCleanUpService.java | 3 ++- dao/src/main/resources/sql/schema-ts-psql.sql | 1 + 10 files changed, 28 insertions(+), 14 deletions(-) diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql index 0916c241a1..41e1cfbb7a 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql @@ -64,6 +64,7 @@ BEGIN AND tablename like 'ts_kv_' || '%' AND tablename != 'ts_kv_latest' AND tablename != 'ts_kv_dictionary' + AND tablename != 'ts_kv_indefinite' LOOP IF partition != partition_by_max_ttl_date THEN IF partition_year IS NOT NULL THEN diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index e281c0958e..01ad8a29e9 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -146,6 +146,11 @@ public class ThingsboardInstallService { databaseTsUpgradeService.upgradeDatabase("2.5.0"); } + case "2.5.4": + log.info("Upgrading ThingsBoard from version 2.5.4 to 2.5.5 ..."); + if (databaseTsUpgradeService != null) { + databaseTsUpgradeService.upgradeDatabase("2.5.4"); + } log.info("Updating system data..."); diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java index 103e8090d9..07b8522323 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java @@ -49,6 +49,7 @@ public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabase log.info("Schema updated."); break; case "2.5.0": + case "2.5.4": break; default: throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java index 7a8174af16..396f84664a 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java @@ -195,6 +195,12 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005001"); } break; + case "2.5.4": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Load Drop Partitions functions ..."); + loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java index d8f7ea61f9..a929a51fb5 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java @@ -177,6 +177,8 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005001"); } break; + case "2.5.4": + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java index 4fc4df0048..e81788958d 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java @@ -38,19 +38,15 @@ public abstract class AbstractCleanUpService { @Value("${spring.datasource.password}") protected String dbPassword; - protected long executeQuery(Connection conn, String query) { - long removed = 0L; - try { - Statement statement = conn.createStatement(); + protected long executeQuery(Connection conn, String query) throws SQLException { + try (Statement statement = conn.createStatement()) { ResultSet resultSet = statement.executeQuery(query); - getWarnings(statement); + if (log.isDebugEnabled()) { + getWarnings(statement); + } resultSet.next(); - removed = resultSet.getLong(1); - log.debug("Successfully executed query: {}", query); - } catch (SQLException e) { - log.debug("Failed to execute query: {} due to: {}", query, e.getMessage()); + return resultSet.getLong(1); } - return removed; } protected void getWarnings(Statement statement) throws SQLException { @@ -65,6 +61,6 @@ public abstract class AbstractCleanUpService { } } - protected abstract void doCleanUp(Connection connection); + protected abstract void doCleanUp(Connection connection) throws SQLException; } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java index 5b094c5c0e..ca52bca7e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -54,7 +54,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { } @Override - protected void doCleanUp(Connection connection) { + protected void doCleanUp(Connection connection) throws SQLException { long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);"); log.info("Total events removed by TTL: [{}]", totalEventsRemoved); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java index cd403ee3b8..2464ab4677 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.PsqlTsDao; import java.sql.Connection; +import java.sql.SQLException; @PsqlTsDao @Service @@ -32,7 +33,7 @@ public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpServi private String partitionType; @Override - protected void doCleanUp(Connection connection) { + protected void doCleanUp(Connection connection) throws SQLException { long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);"); log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved); long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);"); diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java index f5898b9b20..8bdeea46ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.TimescaleDBTsDao; import java.sql.Connection; +import java.sql.SQLException; @TimescaleDBTsDao @Service @@ -28,7 +29,7 @@ import java.sql.Connection; public class TimescaleTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService { @Override - protected void doCleanUp(Connection connection) { + protected void doCleanUp(Connection connection) throws SQLException { long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);"); log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved); } diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 28420a8957..80f111869d 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -105,6 +105,7 @@ BEGIN AND tablename like 'ts_kv_' || '%' AND tablename != 'ts_kv_latest' AND tablename != 'ts_kv_dictionary' + AND tablename != 'ts_kv_indefinite' LOOP IF partition != partition_by_max_ttl_date THEN IF partition_year IS NOT NULL THEN From f16672cbf69717f8506937ec20a87dee4d7052f3 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 2 Sep 2020 18:12:08 +0300 Subject: [PATCH 5/5] added partition property for kafka --- application/src/main/resources/thingsboard.yml | 10 +++++----- .../server/queue/kafka/TbKafkaAdmin.java | 10 +++++++++- docker/queue-kafka.env | 1 + msa/js-executor/config/default.yml | 2 +- msa/js-executor/queue/kafkaTemplate.js | 18 +++++++++++++++--- .../src/main/resources/tb-coap-transport.yml | 10 +++++----- .../src/main/resources/tb-http-transport.yml | 10 +++++----- .../src/main/resources/tb-mqtt-transport.yml | 10 +++++----- 8 files changed, 46 insertions(+), 25 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ac76b67cd3..6400a6bbf0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -617,11 +617,11 @@ queue: security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" other: topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index b92b094af1..bee293bd6d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -37,6 +37,7 @@ public class TbKafkaAdmin implements TbQueueAdmin { private final AdminClient client; private final Map topicConfigs; private final Set topics = ConcurrentHashMap.newKeySet(); + private final int numPartitions; private final short replicationFactor; @@ -50,6 +51,13 @@ public class TbKafkaAdmin implements TbQueueAdmin { log.error("Failed to get all topics.", e); } + String numPartitionsStr = topicConfigs.get("partitions"); + if (numPartitionsStr != null) { + numPartitions = Integer.parseInt(numPartitionsStr); + topicConfigs.remove("partitions"); + } else { + numPartitions = 1; + } replicationFactor = settings.getReplicationFactor(); } @@ -59,7 +67,7 @@ public class TbKafkaAdmin implements TbQueueAdmin { return; } try { - NewTopic newTopic = new NewTopic(topic, 1, replicationFactor).configs(topicConfigs); + NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(topicConfigs); createTopic(newTopic).values().get(topic).get(); topics.add(topic); } catch (ExecutionException ee) { diff --git a/docker/queue-kafka.env b/docker/queue-kafka.env index 63107942fb..0207d64ef5 100644 --- a/docker/queue-kafka.env +++ b/docker/queue-kafka.env @@ -1,2 +1,3 @@ TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 +TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100 diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index fac1ac7e8a..e6ec6f5b36 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -25,7 +25,7 @@ kafka: # Kafka Bootstrap Servers servers: "localhost:9092" replication_factor: "1" - topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" + topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100" use_confluent_cloud: false confluent: sasl: diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 0420173188..2672a8df6a 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -34,7 +34,7 @@ function KafkaProducer() { this.send = async (responseTopic, scriptId, rawResponse, headers) => { if (!topics.includes(responseTopic)) { - let createResponseTopicResult = await createTopic(responseTopic); + let createResponseTopicResult = await createTopic(responseTopic, 1); topics.push(responseTopic); if (createResponseTopicResult) { logger.info('Created new topic: %s', requestTopic); @@ -88,7 +88,18 @@ function KafkaProducer() { kafkaAdmin = kafkaClient.admin(); await kafkaAdmin.connect(); - let createRequestTopicResult = await createTopic(requestTopic); + let partitions = 1; + + for (let i = 0; i < configEntries.length; i++) { + let param = configEntries[i]; + if (param.name === 'partitions') { + partitions = param.value; + configEntries.splice(i, 1); + break; + } + } + + let createRequestTopicResult = await createTopic(requestTopic, partitions); if (createRequestTopicResult) { logger.info('Created new topic: %s', requestTopic); @@ -121,10 +132,11 @@ function KafkaProducer() { } })(); -function createTopic(topic) { +function createTopic(topic, partitions) { return kafkaAdmin.createTopics({ topics: [{ topic: topic, + numPartitions: partitions, replicationFactor: replicationFactor, configEntries: configEntries }] diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index f04a7559b8..3f57b32fab 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -77,11 +77,11 @@ queue: security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" other: topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index a97a58cd85..77d5f30fa7 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -70,11 +70,11 @@ queue: security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" other: topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index b3804e3531..bc18112c8e 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -98,11 +98,11 @@ queue: security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" other: topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}"