From e5311984bba0d6449e6db65c9fdfac22cf2e98b7 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 7 Feb 2024 15:24:40 +0200 Subject: [PATCH] Housekeeper API --- .../DefaultHousekeeperService.java | 144 ++++++++++++++++++ .../HousekeeperReprocessingService.java | 134 ++++++++++++++++ .../AttributesDeletionTaskProcessor.java | 41 +++++ .../EntityDeletionTaskProcessor.java | 37 +++++ .../EventsDeletionTaskProcessor.java | 38 +++++ .../processor/HousekeeperTaskProcessor.java | 27 ++++ .../TelemetryDeletionTaskProcessor.java | 41 +++++ .../src/main/resources/thingsboard.yml | 7 + common/proto/src/main/proto/queue.proto | 8 + .../queue/common/AbstractTbQueueTemplate.java | 4 +- .../queue/kafka/TbKafkaTopicConfigs.java | 5 + .../provider/AwsSqsMonolithQueueFactory.java | 21 +++ .../provider/AwsSqsTbCoreQueueFactory.java | 21 +++ .../InMemoryMonolithQueueFactory.java | 20 +++ .../provider/KafkaMonolithQueueFactory.java | 51 ++++++- .../provider/KafkaTbCoreQueueFactory.java | 50 ++++++ .../provider/PubSubMonolithQueueFactory.java | 21 +++ .../provider/PubSubTbCoreQueueFactory.java | 21 +++ .../RabbitMqMonolithQueueFactory.java | 21 +++ .../provider/RabbitMqTbCoreQueueFactory.java | 20 +++ .../ServiceBusMonolithQueueFactory.java | 21 +++ .../ServiceBusTbCoreQueueFactory.java | 21 +++ .../queue/provider/TbCoreQueueFactory.java | 11 ++ .../provider/TbCoreQueueProducerProvider.java | 17 +++ .../provider/TbQueueProducerProvider.java | 6 + .../TbRuleEngineProducerProvider.java | 11 ++ .../TbTransportQueueProducerProvider.java | 10 ++ .../TbVersionControlProducerProvider.java | 10 ++ .../queue/settings/TbQueueCoreSettings.java | 6 + .../server/dao/device/DeviceServiceImpl.java | 5 + .../dao/entity/AbstractEntityService.java | 6 + .../dao/housekeeper/HousekeeperService.java | 24 +++ .../dao/housekeeper/data/HousekeeperTask.java | 51 +++++++ .../housekeeper/data/HousekeeperTaskType.java | 23 +++ 34 files changed, 951 insertions(+), 3 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java new file mode 100644 index 0000000000..7c7e72b511 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.housekeeper.HousekeeperService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@TbCoreComponent +@Service +@Slf4j +public class DefaultHousekeeperService implements HousekeeperService { + + private final Map taskProcessors; + + private final TbQueueConsumer> consumer; + private final TbQueueProducer> producer; + private final HousekeeperReprocessingService reprocessingService; + private final DataDecodingEncodingService dataDecodingEncodingService; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); + + @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + private int pollInterval; + + private boolean stopped; + + public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService, + TbCoreQueueFactory queueFactory, + TbQueueProducerProvider producerProvider, + DataDecodingEncodingService dataDecodingEncodingService, List taskProcessors) { + this.consumer = queueFactory.createHousekeeperMsgConsumer(); + this.producer = producerProvider.getHousekeeperMsgProducer(); + this.reprocessingService = reprocessingService; + this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); + this.dataDecodingEncodingService = dataDecodingEncodingService; + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + consumer.subscribe(); + consumerExecutor.submit(() -> { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(pollInterval); + if (msgs.isEmpty()) { + continue; + } + + for (TbProtoQueueMsg msg : msgs) { + try { + processTask(msg); + } catch (Exception e) { + log.error("Message processing failed", e); + } + } + consumer.commit(); + } catch (Throwable t) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", t); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException interruptedException) { + log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); + } + } + } + } + }); + log.info("Started Housekeeper service"); + } + + protected void processTask(TbProtoQueueMsg msg) { + HousekeeperTask task = dataDecodingEncodingService.decode(msg.getValue().getTask().getValue().toByteArray()).get(); + HousekeeperTaskProcessor taskProcessor = taskProcessors.get(task.getTaskType()); + if (taskProcessor == null) { + log.error("Unsupported task type {}: {}", task.getTaskType(), task); + return; + } + + log.info("[{}] Processing task: {}", task.getTenantId(), task); + try { + taskProcessor.process(task); + } catch (Exception e) { + log.error("[{}] Task processing failed: {}", task.getTenantId(), task, e); + reprocessingService.submitForReprocessing(msg); + } + } + + @Override + public void submitTask(HousekeeperTask task) { + TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), ToHousekeeperServiceMsg.newBuilder() + .setTask(HousekeeperTaskProto.newBuilder() + .setValue(ByteString.copyFrom(dataDecodingEncodingService.encode(task))) + .build()) + .build()), null); + } + + @PreDestroy + private void stop() { + log.info("Stopped Housekeeper service"); + stopped = true; + consumer.unsubscribe(); + consumerExecutor.shutdownNow(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java new file mode 100644 index 0000000000..234cd07c36 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -0,0 +1,134 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToLong; +import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.longToBytes; + +@TbCoreComponent +@Service +@Slf4j +public class HousekeeperReprocessingService { + + private final DefaultHousekeeperService housekeeperService; + private final TbQueueProducerProvider producerProvider; + private final TbQueueConsumer> consumer; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); + + @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + private int pollInterval; + + private final long startTs = System.currentTimeMillis(); + private boolean stopped; + // todo: stats + + public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService, + TbCoreQueueFactory queueFactory, + TbQueueProducerProvider producerProvider) { + this.housekeeperService = housekeeperService; + this.consumer = queueFactory.createHousekeeperDelayedMsgConsumer(); + this.producerProvider = producerProvider; + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + consumer.subscribe(); + consumerExecutor.submit(() -> { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(pollInterval); + if (msgs.isEmpty()) { + stop(); + return; + } + + for (TbProtoQueueMsg msg : msgs) { + long msgTs = Uuids.unixTimestamp(msg.getKey()); + if (msgTs >= startTs) { + stop(); + return; // fixme: we should commit already reprocessed messages + } + + try { + reprocessTask(msg); + } catch (Exception e) { + log.error("Message processing failed", e); + } + } + consumer.commit(); + } catch (Throwable t) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", t); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException interruptedException) { + log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); + } + } + } + } + }); + log.info("Started Housekeeper tasks reprocessing"); + } + + private void reprocessTask(TbProtoQueueMsg msg) { + housekeeperService.processTask(msg);// fixme: or should we submit to queue? + } + + public void submitForReprocessing(TbProtoQueueMsg msg) { + TbQueueMsgHeaders msgHeaders = msg.getHeaders(); + long reprocessingAttempts = Optional.ofNullable(msgHeaders.get("reprocessingAttempts")) + .map(header -> bytesToLong(header)) + .orElse(0L); + reprocessingAttempts++; + msgHeaders.put("reprocessingAttempts", longToBytes(reprocessingAttempts)); + + var producer = producerProvider.getHousekeeperDelayedMsgProducer(); + TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), msg.getValue(), msgHeaders), null); + } + + @PreDestroy + private void stop() { + log.info("Stopped Housekeeper tasks reprocessing"); + stopped = true; + consumer.unsubscribe(); + consumerExecutor.shutdownNow(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java new file mode 100644 index 0000000000..34bd95a901 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { + + private final AttributesService attributesService; + + @Override + public void process(HousekeeperTask task) throws Exception { +// attributesService.removeAll(task.getTenantId(), task.getEntityId(), DataConstants.CLIENT_SCOPE); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ATTRIBUTES; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java new file mode 100644 index 0000000000..e1b0877d2c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { + + @Override + public void process(HousekeeperTask task) throws Exception { + + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java new file mode 100644 index 0000000000..7b8fdb1242 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { + private final EventService eventService; + + @Override + public void process(HousekeeperTask task) throws Exception { + eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_EVENTS; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java new file mode 100644 index 0000000000..bfa7466dc3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +public interface HousekeeperTaskProcessor { + + void process(HousekeeperTask task) throws Exception; + + HousekeeperTaskType getTaskType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java new file mode 100644 index 0000000000..feb5b0ddc1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +@Component +@RequiredArgsConstructor +public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { + + private final TimeseriesService timeseriesService; + + @Override + public void process(HousekeeperTask task) throws Exception { +// timeseriesService.remove() +// timeseriesService.removeAllLatest(); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ATTRIBUTES; + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 70a5146bb7..17b4c85cfb 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1383,6 +1383,9 @@ queue: # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + tb_housekeeper: + - key: max.poll.records + value: "1" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms @@ -1404,6 +1407,7 @@ queue: ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Version Control topic version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + housekeeper: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1559,6 +1563,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + housekeeper: + topic: "tb_housekeeper" + poll-interval-ms: "1000" vc: # Default topic name for Kafka, RabbitMQ, etc. topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4da2883291..0dbd96a53d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1391,3 +1391,11 @@ message LifecycleEventProto { bool success = 7; string error = 8; } + +message ToHousekeeperServiceMsg { + HousekeeperTaskProto task = 1; +} + +message HousekeeperTaskProto { + bytes value = 1; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index f11eaaef48..9340cafa67 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -46,13 +46,13 @@ public class AbstractTbQueueTemplate { return new String(data, StandardCharsets.UTF_8); } - protected static byte[] longToBytes(long x) { + public static byte[] longToBytes(long x) { ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); longBuffer.putLong(0, x); return longBuffer.array(); } - protected static long bytesToLong(byte[] bytes) { + public static long bytesToLong(byte[] bytes) { return ByteBuffer.wrap(bytes).getLong(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index d23d92ee26..d3a5eb821d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -43,6 +43,8 @@ public class TbKafkaTopicConfigs { private String fwUpdatesProperties; @Value("${queue.kafka.topic-properties.version-control:}") private String vcProperties; + @Value("${queue.kafka.topic-properties.housekeeper:}") + private String housekeeperProperties; @Getter private Map coreConfigs; @@ -62,6 +64,8 @@ public class TbKafkaTopicConfigs { private Map fwUpdatesConfigs; @Getter private Map vcConfigs; + @Getter + private Map housekeeperConfigs; @PostConstruct private void init() { @@ -76,6 +80,7 @@ public class TbKafkaTopicConfigs { jsExecutorResponseConfigs.put(NUM_PARTITIONS_SETTING, "1"); fwUpdatesConfigs = PropertyUtils.getProps(fwUpdatesProperties); vcConfigs = PropertyUtils.getProps(vcProperties); + housekeeperConfigs = PropertyUtils.getProps(housekeeperProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 8a127384be..5a61bc1d80 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -226,6 +227,26 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 6c69b2ec10..a1be25f29d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -205,6 +206,26 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index a681038aa9..1ee8b404a5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -160,6 +160,26 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 662d29af5b..5ee9b6f53b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -39,8 +40,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -82,6 +83,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin housekeeperAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -115,6 +117,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -346,6 +349,52 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .clientId("monolith-housekeeper-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("monolith-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-delayed-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index db6ee52181..c01ba32f70 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -80,6 +81,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin housekeeperAdmin; public KafkaTbCoreQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -112,6 +114,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -303,6 +306,53 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-core-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .clientId("tb-core-housekeeper-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-core-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("tb-core-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-delayed-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 62fe40f853..a63a9fc16b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -227,6 +228,26 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic()); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 0e7e1ba241..5f1f282e39 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -198,6 +199,26 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return null; } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 1b7f705175..802b08773d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -224,6 +225,26 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new TbRabbitMqProducerTemplate<>(vcAdmin, rabbitMqSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index e0d0ea3b3a..ceb6e03ca5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -198,6 +198,26 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 359cbf67b8..1e9c1cadaf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -223,6 +224,26 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return new TbServiceBusProducerTemplate<>(vcAdmin, serviceBusSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index d2ef072ffa..e39df45b82 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -198,6 +199,26 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return null; } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 5aa7866033..279a469415 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -16,8 +16,10 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -130,4 +132,13 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { * @return */ TbQueueProducer> createVersionControlMsgProducer(); + + TbQueueProducer> createHousekeeperMsgProducer(); + + TbQueueConsumer> createHousekeeperMsgConsumer(); + + TbQueueProducer> createHousekeeperDelayedMsgProducer(); + + TbQueueConsumer> createHousekeeperDelayedMsgConsumer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 5a99f361b4..4fa22cb7e3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -16,8 +16,10 @@ package org.thingsboard.server.queue.provider; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -41,6 +43,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; + private TbQueueProducer> toHousekeeper; + private TbQueueProducer> toHousekeeperDelayed; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -55,6 +59,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); + this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); + this.toHousekeeperDelayed = tbQueueProvider.createHousekeeperDelayedMsgProducer(); } @Override @@ -91,4 +97,15 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbVersionControlMsgProducer() { return toVersionControl; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return toHousekeeper; + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + return toHousekeeperDelayed; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 4dbdca4375..5cdf390f47 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -78,4 +79,9 @@ public interface TbQueueProducerProvider { * @return */ TbQueueProducer> getTbVersionControlMsgProducer(); + + TbQueueProducer> getHousekeeperMsgProducer(); + + TbQueueProducer> getHousekeeperDelayedMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index e4d98796f4..c5ddf57fc0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -89,4 +89,15 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbVersionControlMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return null; // fixme + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index c83b905a7c..e8dd7431a6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -85,4 +85,14 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index dbbaa9f072..7ddd2109ba 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -81,4 +81,14 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 0a8c12eb8d..4edb688235 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -34,6 +34,12 @@ public class TbQueueCoreSettings { @Value("${queue.core.usage-stats-topic:tb_usage_stats}") private String usageStatsTopic; + @Value("${queue.core.housekeeper.topic:tb_housekeeper}") + private String housekeeperTopic; + + @Value("${queue.core.housekeeper.topic:tb_housekeeper.delayed}") + private String housekeeperDelayedTopic; + @Value("${queue.core.partitions}") private int partitions; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index f190923167..f0e5dc1a4a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -75,6 +75,7 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.IncorrectParameterException; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -328,6 +329,10 @@ public class DeviceServiceImpl extends AbstractCachedEntityService