Browse Source

Housekeeper API

pull/10201/head
ViacheslavKlimov 2 years ago
parent
commit
e5311984bb
  1. 144
      application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java
  2. 134
      application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java
  3. 41
      application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java
  4. 37
      application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java
  5. 38
      application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java
  6. 27
      application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java
  7. 41
      application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java
  8. 7
      application/src/main/resources/thingsboard.yml
  9. 8
      common/proto/src/main/proto/queue.proto
  10. 4
      common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java
  11. 5
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java
  12. 21
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java
  13. 21
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java
  14. 20
      common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java
  15. 51
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java
  16. 50
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java
  17. 21
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java
  18. 21
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java
  19. 21
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java
  20. 20
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java
  21. 21
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java
  22. 21
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java
  23. 11
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java
  24. 17
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java
  25. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java
  26. 11
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java
  27. 10
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java
  28. 10
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java
  29. 6
      common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java
  30. 5
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java
  31. 6
      dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java
  32. 24
      dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java
  33. 51
      dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java
  34. 23
      dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java

144
application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java

@ -0,0 +1,144 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.housekeeper;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.housekeeper.HousekeeperService;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType;
import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor;
import javax.annotation.PreDestroy;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
@TbCoreComponent
@Service
@Slf4j
public class DefaultHousekeeperService implements HousekeeperService {
private final Map<HousekeeperTaskType, HousekeeperTaskProcessor> taskProcessors;
private final TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> consumer;
private final TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> producer;
private final HousekeeperReprocessingService reprocessingService;
private final DataDecodingEncodingService dataDecodingEncodingService;
private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer"));
@Value("${queue.core.housekeeper.poll-interval-ms:10000}")
private int pollInterval;
private boolean stopped;
public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService,
TbCoreQueueFactory queueFactory,
TbQueueProducerProvider producerProvider,
DataDecodingEncodingService dataDecodingEncodingService, List<HousekeeperTaskProcessor> taskProcessors) {
this.consumer = queueFactory.createHousekeeperMsgConsumer();
this.producer = producerProvider.getHousekeeperMsgProducer();
this.reprocessingService = reprocessingService;
this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p));
this.dataDecodingEncodingService = dataDecodingEncodingService;
}
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void afterStartUp() {
consumer.subscribe();
consumerExecutor.submit(() -> {
while (!stopped && !consumer.isStopped()) {
try {
List<TbProtoQueueMsg<ToHousekeeperServiceMsg>> msgs = consumer.poll(pollInterval);
if (msgs.isEmpty()) {
continue;
}
for (TbProtoQueueMsg<ToHousekeeperServiceMsg> msg : msgs) {
try {
processTask(msg);
} catch (Exception e) {
log.error("Message processing failed", e);
}
}
consumer.commit();
} catch (Throwable t) {
if (!consumer.isStopped()) {
log.warn("Failed to process messages from queue", t);
try {
Thread.sleep(pollInterval);
} catch (InterruptedException interruptedException) {
log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException);
}
}
}
}
});
log.info("Started Housekeeper service");
}
protected void processTask(TbProtoQueueMsg<ToHousekeeperServiceMsg> msg) {
HousekeeperTask task = dataDecodingEncodingService.<HousekeeperTask>decode(msg.getValue().getTask().getValue().toByteArray()).get();
HousekeeperTaskProcessor taskProcessor = taskProcessors.get(task.getTaskType());
if (taskProcessor == null) {
log.error("Unsupported task type {}: {}", task.getTaskType(), task);
return;
}
log.info("[{}] Processing task: {}", task.getTenantId(), task);
try {
taskProcessor.process(task);
} catch (Exception e) {
log.error("[{}] Task processing failed: {}", task.getTenantId(), task, e);
reprocessingService.submitForReprocessing(msg);
}
}
@Override
public void submitTask(HousekeeperTask task) {
TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build();
producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), ToHousekeeperServiceMsg.newBuilder()
.setTask(HousekeeperTaskProto.newBuilder()
.setValue(ByteString.copyFrom(dataDecodingEncodingService.encode(task)))
.build())
.build()), null);
}
@PreDestroy
private void stop() {
log.info("Stopped Housekeeper service");
stopped = true;
consumer.unsubscribe();
consumerExecutor.shutdownNow();
}
}

134
application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java

@ -0,0 +1,134 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.housekeeper;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueMsgHeaders;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.TbCoreComponent;
import javax.annotation.PreDestroy;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToLong;
import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.longToBytes;
@TbCoreComponent
@Service
@Slf4j
public class HousekeeperReprocessingService {
private final DefaultHousekeeperService housekeeperService;
private final TbQueueProducerProvider producerProvider;
private final TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> consumer;
private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer"));
@Value("${queue.core.housekeeper.poll-interval-ms:10000}")
private int pollInterval;
private final long startTs = System.currentTimeMillis();
private boolean stopped;
// todo: stats
public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService,
TbCoreQueueFactory queueFactory,
TbQueueProducerProvider producerProvider) {
this.housekeeperService = housekeeperService;
this.consumer = queueFactory.createHousekeeperDelayedMsgConsumer();
this.producerProvider = producerProvider;
}
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void afterStartUp() {
consumer.subscribe();
consumerExecutor.submit(() -> {
while (!stopped && !consumer.isStopped()) {
try {
List<TbProtoQueueMsg<ToHousekeeperServiceMsg>> msgs = consumer.poll(pollInterval);
if (msgs.isEmpty()) {
stop();
return;
}
for (TbProtoQueueMsg<ToHousekeeperServiceMsg> msg : msgs) {
long msgTs = Uuids.unixTimestamp(msg.getKey());
if (msgTs >= startTs) {
stop();
return; // fixme: we should commit already reprocessed messages
}
try {
reprocessTask(msg);
} catch (Exception e) {
log.error("Message processing failed", e);
}
}
consumer.commit();
} catch (Throwable t) {
if (!consumer.isStopped()) {
log.warn("Failed to process messages from queue", t);
try {
Thread.sleep(pollInterval);
} catch (InterruptedException interruptedException) {
log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException);
}
}
}
}
});
log.info("Started Housekeeper tasks reprocessing");
}
private void reprocessTask(TbProtoQueueMsg<ToHousekeeperServiceMsg> msg) {
housekeeperService.processTask(msg);// fixme: or should we submit to queue?
}
public void submitForReprocessing(TbProtoQueueMsg<ToHousekeeperServiceMsg> msg) {
TbQueueMsgHeaders msgHeaders = msg.getHeaders();
long reprocessingAttempts = Optional.ofNullable(msgHeaders.get("reprocessingAttempts"))
.map(header -> bytesToLong(header))
.orElse(0L);
reprocessingAttempts++;
msgHeaders.put("reprocessingAttempts", longToBytes(reprocessingAttempts));
var producer = producerProvider.getHousekeeperDelayedMsgProducer();
TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build();
producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), msg.getValue(), msgHeaders), null);
}
@PreDestroy
private void stop() {
log.info("Stopped Housekeeper tasks reprocessing");
stopped = true;
consumer.unsubscribe();
consumerExecutor.shutdownNow();
}
}

41
application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java

@ -0,0 +1,41 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.housekeeper.processor;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType;
@Component
@RequiredArgsConstructor
public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor {
private final AttributesService attributesService;
@Override
public void process(HousekeeperTask task) throws Exception {
// attributesService.removeAll(task.getTenantId(), task.getEntityId(), DataConstants.CLIENT_SCOPE);
}
@Override
public HousekeeperTaskType getTaskType() {
return HousekeeperTaskType.DELETE_ATTRIBUTES;
}
}

37
application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.housekeeper.processor;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType;
@Component
@RequiredArgsConstructor
public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor {
@Override
public void process(HousekeeperTask task) throws Exception {
}
@Override
public HousekeeperTaskType getTaskType() {
return HousekeeperTaskType.DELETE_ENTITY;
}
}

38
application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java

@ -0,0 +1,38 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.housekeeper.processor;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import org.thingsboard.server.dao.event.EventService;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType;
@Component
@RequiredArgsConstructor
public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor {
private final EventService eventService;
@Override
public void process(HousekeeperTask task) throws Exception {
eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis());
}
@Override
public HousekeeperTaskType getTaskType() {
return HousekeeperTaskType.DELETE_EVENTS;
}
}

27
application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java

@ -0,0 +1,27 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.housekeeper.processor;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType;
public interface HousekeeperTaskProcessor {
void process(HousekeeperTask task) throws Exception;
HousekeeperTaskType getTaskType();
}

41
application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java

@ -0,0 +1,41 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.housekeeper.processor;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
@Component
@RequiredArgsConstructor
public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor {
private final TimeseriesService timeseriesService;
@Override
public void process(HousekeeperTask task) throws Exception {
// timeseriesService.remove()
// timeseriesService.removeAllLatest();
}
@Override
public HousekeeperTaskType getTaskType() {
return HousekeeperTaskType.DELETE_ATTRIBUTES;
}
}

7
application/src/main/resources/thingsboard.yml

@ -1383,6 +1383,9 @@ queue:
# tb_rule_engine.sq:
# - key: max.poll.records
# value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}"
tb_housekeeper:
- key: max.poll.records
value: "1"
other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000"
other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside
# - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms
@ -1404,6 +1407,7 @@ queue:
ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}"
# Kafka properties for Version Control topic
version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
housekeeper: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1"
consumer-stats:
# Prints lag between consumer group offset and last messages offset in Kafka topics
enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}"
@ -1559,6 +1563,9 @@ queue:
enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}"
# Statistics printing interval for Core microservices
print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}"
housekeeper:
topic: "tb_housekeeper"
poll-interval-ms: "1000"
vc:
# Default topic name for Kafka, RabbitMQ, etc.
topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}"

8
common/proto/src/main/proto/queue.proto

@ -1391,3 +1391,11 @@ message LifecycleEventProto {
bool success = 7;
string error = 8;
}
message ToHousekeeperServiceMsg {
HousekeeperTaskProto task = 1;
}
message HousekeeperTaskProto {
bytes value = 1;
}

4
common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java

@ -46,13 +46,13 @@ public class AbstractTbQueueTemplate {
return new String(data, StandardCharsets.UTF_8);
}
protected static byte[] longToBytes(long x) {
public static byte[] longToBytes(long x) {
ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES);
longBuffer.putLong(0, x);
return longBuffer.array();
}
protected static long bytesToLong(byte[] bytes) {
public static long bytesToLong(byte[] bytes) {
return ByteBuffer.wrap(bytes).getLong();
}
}

5
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java

@ -43,6 +43,8 @@ public class TbKafkaTopicConfigs {
private String fwUpdatesProperties;
@Value("${queue.kafka.topic-properties.version-control:}")
private String vcProperties;
@Value("${queue.kafka.topic-properties.housekeeper:}")
private String housekeeperProperties;
@Getter
private Map<String, String> coreConfigs;
@ -62,6 +64,8 @@ public class TbKafkaTopicConfigs {
private Map<String, String> fwUpdatesConfigs;
@Getter
private Map<String, String> vcConfigs;
@Getter
private Map<String, String> housekeeperConfigs;
@PostConstruct
private void init() {
@ -76,6 +80,7 @@ public class TbKafkaTopicConfigs {
jsExecutorResponseConfigs.put(NUM_PARTITIONS_SETTING, "1");
fwUpdatesConfigs = PropertyUtils.getProps(fwUpdatesProperties);
vcConfigs = PropertyUtils.getProps(vcProperties);
housekeeperConfigs = PropertyUtils.getProps(housekeeperProperties);
}
}

21
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -226,6 +227,26 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
return null;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {

21
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java

@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -205,6 +206,26 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory {
return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
return null;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {

20
common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java

@ -160,6 +160,26 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(vcSettings.getTopic()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()));
}
@Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}")
private void printInMemoryStats() {
storage.printStats();

51
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java

@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -39,8 +40,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
@ -82,6 +83,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
private final TbQueueAdmin notificationAdmin;
private final TbQueueAdmin fwUpdatesAdmin;
private final TbQueueAdmin vcAdmin;
private final TbQueueAdmin housekeeperAdmin;
private final AtomicLong consumerCount = new AtomicLong();
@ -115,6 +117,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs());
this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs());
this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs());
this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs());
}
@Override
@ -346,6 +349,52 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
return requestBuilder.build();
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
return TbKafkaProducerTemplate.<TbProtoQueueMsg<ToHousekeeperServiceMsg>>builder()
.settings(kafkaSettings)
.clientId("monolith-housekeeper-producer-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic()))
.admin(housekeeperAdmin)
.build();
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
return TbKafkaConsumerTemplate.<TbProtoQueueMsg<ToHousekeeperServiceMsg>>builder()
.settings(kafkaSettings)
.topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic()))
.clientId("monolith-housekeeper-consumer-" + serviceInfoProvider.getServiceId())
.groupId(topicService.buildTopicName("monolith-housekeeper-consumer"))
.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders()))
.admin(housekeeperAdmin)
.statsService(consumerStatsService)
.build();
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
return TbKafkaProducerTemplate.<TbProtoQueueMsg<ToHousekeeperServiceMsg>>builder()
.settings(kafkaSettings)
.clientId("monolith-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()))
.admin(housekeeperAdmin)
.build();
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
return TbKafkaConsumerTemplate.<TbProtoQueueMsg<ToHousekeeperServiceMsg>>builder()
.settings(kafkaSettings)
.topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()))
.clientId("monolith-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId())
.groupId(topicService.buildTopicName("monolith-housekeeper-delayed-consumer"))
.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders()))
.admin(housekeeperAdmin)
.statsService(consumerStatsService)
.build();
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {

50
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java

@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
@ -80,6 +81,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
private final TbQueueAdmin notificationAdmin;
private final TbQueueAdmin fwUpdatesAdmin;
private final TbQueueAdmin vcAdmin;
private final TbQueueAdmin housekeeperAdmin;
public KafkaTbCoreQueueFactory(TopicService topicService,
TbKafkaSettings kafkaSettings,
@ -112,6 +114,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs());
this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs());
this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs());
this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs());
}
@Override
@ -303,6 +306,53 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
return requestBuilder.build();
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
return TbKafkaProducerTemplate.<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>>builder()
.settings(kafkaSettings)
.clientId("tb-core-housekeeper-producer-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic()))
.admin(housekeeperAdmin)
.build();
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
return TbKafkaConsumerTemplate.<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>>builder()
.settings(kafkaSettings)
.topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic()))
.clientId("tb-core-housekeeper-consumer-" + serviceInfoProvider.getServiceId())
.groupId(topicService.buildTopicName("tb-core-housekeeper-consumer"))
.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders()))
.admin(housekeeperAdmin)
.statsService(consumerStatsService)
.build();
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
return TbKafkaProducerTemplate.<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>>builder()
.settings(kafkaSettings)
.clientId("tb-core-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()))
.admin(housekeeperAdmin)
.build();
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
return TbKafkaConsumerTemplate.<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>>builder()
.settings(kafkaSettings)
.topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()))
.clientId("tb-core-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId())
.groupId(topicService.buildTopicName("tb-core-housekeeper-delayed-consumer"))
.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders()))
.admin(housekeeperAdmin)
.statsService(consumerStatsService)
.build();
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {

21
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -227,6 +228,26 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
return null;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {

21
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java

@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -198,6 +199,26 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
return null;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
return null;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {

21
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -224,6 +225,26 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
return new TbRabbitMqProducerTemplate<>(vcAdmin, rabbitMqSettings, topicService.buildTopicName(vcSettings.getTopic()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
return null;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {

20
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java

@ -198,6 +198,26 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
return null;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {

21
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java

@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -223,6 +224,26 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
return new TbServiceBusProducerTemplate<>(vcAdmin, serviceBusSettings, topicService.buildTopicName(vcSettings.getTopic()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
return null;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {

21
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java

@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -198,6 +199,26 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
return null;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
return null;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
return null;
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {

11
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java

@ -16,8 +16,10 @@
package org.thingsboard.server.queue.provider;
import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -130,4 +132,13 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory {
* @return
*/
TbQueueProducer<TbProtoQueueMsg<ToVersionControlServiceMsg>> createVersionControlMsgProducer();
TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgProducer();
TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer();
TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer();
TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer();
}

17
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java

@ -16,8 +16,10 @@
package org.thingsboard.server.queue.provider;
import org.springframework.stereotype.Service;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
@ -41,6 +43,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider {
private TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toTbCoreNotifications;
private TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> toUsageStats;
private TbQueueProducer<TbProtoQueueMsg<ToVersionControlServiceMsg>> toVersionControl;
private TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> toHousekeeper;
private TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> toHousekeeperDelayed;
public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) {
this.tbQueueProvider = tbQueueProvider;
@ -55,6 +59,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider {
this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer();
this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer();
this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer();
this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer();
this.toHousekeeperDelayed = tbQueueProvider.createHousekeeperDelayedMsgProducer();
}
@Override
@ -91,4 +97,15 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider {
public TbQueueProducer<TbProtoQueueMsg<ToVersionControlServiceMsg>> getTbVersionControlMsgProducer() {
return toVersionControl;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> getHousekeeperMsgProducer() {
return toHousekeeper;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> getHousekeeperDelayedMsgProducer() {
return toHousekeeperDelayed;
}
}

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java

@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
@ -78,4 +79,9 @@ public interface TbQueueProducerProvider {
* @return
*/
TbQueueProducer<TbProtoQueueMsg<ToVersionControlServiceMsg>> getTbVersionControlMsgProducer();
TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> getHousekeeperMsgProducer();
TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> getHousekeeperDelayedMsgProducer();
}

11
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java

@ -89,4 +89,15 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider {
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToVersionControlServiceMsg>> getTbVersionControlMsgProducer() {
throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!");
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperMsgProducer() {
return null; // fixme
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperDelayedMsgProducer() {
throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!");
}
}

10
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java

@ -85,4 +85,14 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> getTbUsageStatsMsgProducer() {
return toUsageStats;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperMsgProducer() {
throw new RuntimeException("Not Implemented! Should not be used by Transport!");
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperDelayedMsgProducer() {
throw new RuntimeException("Not Implemented! Should not be used by Transport!");
}
}

10
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java

@ -81,4 +81,14 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> getTbUsageStatsMsgProducer() {
return toUsageStats;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperMsgProducer() {
throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!");
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperDelayedMsgProducer() {
throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!");
}
}

6
common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java

@ -34,6 +34,12 @@ public class TbQueueCoreSettings {
@Value("${queue.core.usage-stats-topic:tb_usage_stats}")
private String usageStatsTopic;
@Value("${queue.core.housekeeper.topic:tb_housekeeper}")
private String housekeeperTopic;
@Value("${queue.core.housekeeper.topic:tb_housekeeper.delayed}")
private String housekeeperDelayedTopic;
@Value("${queue.core.partitions}")
private int partitions;
}

5
dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java

@ -75,6 +75,7 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover;
@ -328,6 +329,10 @@ public class DeviceServiceImpl extends AbstractCachedEntityService<DeviceCacheKe
log.trace("Executing deleteDevice [{}]", device.getId());
deviceCredentialsService.deleteDeviceCredentialsByDeviceId(tenantId, device.getId());
relationService.deleteEntityRelations(tenantId, device.getId());
housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, device.getId()));
housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, device.getId()));
housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, device.getId()));
// todo: extract to cleanUpRelatedData
deviceDao.removeById(tenantId, device.getUuidId());

6
dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java

@ -31,6 +31,9 @@ import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.housekeeper.HouseKeeperService;
import org.thingsboard.server.dao.housekeeper.HousekeeperService;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask;
import org.thingsboard.server.dao.relation.RelationService;
import java.util.Collections;
@ -63,6 +66,9 @@ public abstract class AbstractEntityService {
@Autowired(required = false)
protected EdgeService edgeService;
@Autowired
protected HousekeeperService housekeeperService;
protected void createRelation(TenantId tenantId, EntityRelation relation) {
log.debug("Creating relation: {}", relation);
relationService.saveRelation(tenantId, relation);

24
dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java

@ -0,0 +1,24 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.housekeeper;
import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask;
public interface HousekeeperService {
void submitTask(HousekeeperTask task);
}

51
dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java

@ -0,0 +1,51 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.housekeeper.data;
import lombok.Data;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import java.io.Serializable;
/*
* on start, read the retry queue and put the messages back to main queue (save offset)
* */
@Data
public class HousekeeperTask implements Serializable {
private final TenantId tenantId;
private final EntityId entityId;
private final HousekeeperTaskType taskType;
// maybe we should not delete relations asynchronously
// public static HousekeeperTask deleteRelations(TenantId tenantId, EntityId entityId) {
// return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_RELATIONS);
// }
public static HousekeeperTask deleteAttributes(TenantId tenantId, EntityId entityId) {
return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ATTRIBUTES);
}
public static HousekeeperTask deleteTelemetry(TenantId tenantId, EntityId entityId) {
return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_TELEMETRY);
}
public static HousekeeperTask deleteEvents(TenantId tenantId, EntityId entityId) {
return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_EVENTS);
}
}

23
dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java

@ -0,0 +1,23 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.housekeeper.data;
public enum HousekeeperTaskType {
DELETE_ENTITY,
DELETE_ATTRIBUTES,
DELETE_TELEMETRY, // maybe divide into latest and ts kv history?
DELETE_EVENTS
}
Loading…
Cancel
Save