Andrii Landiak
2 years ago
29 changed files with 228 additions and 73 deletions
@ -0,0 +1,111 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2024 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.ttl; |
||||
|
|
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
||||
|
import org.springframework.scheduling.annotation.Scheduled; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
||||
|
import org.thingsboard.server.common.data.AttributeScope; |
||||
|
import org.thingsboard.server.common.data.edge.Edge; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
||||
|
import org.thingsboard.server.common.data.page.PageDataIterable; |
||||
|
import org.thingsboard.server.common.msg.queue.ServiceType; |
||||
|
import org.thingsboard.server.dao.attributes.AttributesService; |
||||
|
import org.thingsboard.server.dao.edge.EdgeService; |
||||
|
import org.thingsboard.server.dao.tenant.TenantService; |
||||
|
import org.thingsboard.server.queue.discovery.PartitionService; |
||||
|
import org.thingsboard.server.queue.discovery.TopicService; |
||||
|
import org.thingsboard.server.queue.kafka.TbKafkaAdmin; |
||||
|
import org.thingsboard.server.queue.kafka.TbKafkaSettings; |
||||
|
import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; |
||||
|
import org.thingsboard.server.queue.util.TbCoreComponent; |
||||
|
import org.thingsboard.server.service.state.DefaultDeviceStateService; |
||||
|
|
||||
|
import java.time.Instant; |
||||
|
import java.util.Date; |
||||
|
import java.util.Optional; |
||||
|
import java.util.concurrent.ExecutorService; |
||||
|
import java.util.concurrent.Executors; |
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
|
||||
|
@Slf4j |
||||
|
@Service |
||||
|
@TbCoreComponent |
||||
|
@RequiredArgsConstructor |
||||
|
@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ${edges.enabled:true}") |
||||
|
public class KafkaEdgeTopicsCleanUpService { |
||||
|
|
||||
|
private static final long ONE_MONTH_MILLIS = TimeUnit.DAYS.toChronoUnit().getDuration().multipliedBy(30).toMillis(); |
||||
|
|
||||
|
private final EdgeService edgeService; |
||||
|
private final TenantService tenantService; |
||||
|
private final AttributesService attributesService; |
||||
|
|
||||
|
private final TopicService topicService; |
||||
|
private final PartitionService partitionService; |
||||
|
|
||||
|
private final TbKafkaSettings kafkaSettings; |
||||
|
private final TbKafkaTopicConfigs kafkaTopicConfigs; |
||||
|
|
||||
|
private final ExecutorService executorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("kafka-edge-topic-cleanup")); |
||||
|
|
||||
|
@Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.edge_events.checking_interval})}", fixedDelayString = "${sql.ttl.edge_events.checking_interval}") |
||||
|
public void cleanUp() { |
||||
|
executorService.submit(() -> { |
||||
|
PageDataIterable<TenantId> tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); |
||||
|
for (TenantId tenantId : tenants) { |
||||
|
try { |
||||
|
cleanUp(tenantId); |
||||
|
} catch (Exception e) { |
||||
|
log.warn("Failed to drop kafka topics for tenant {}", tenantId, e); |
||||
|
} |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
private void cleanUp(TenantId tenantId) throws Exception { |
||||
|
if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
PageDataIterable<Edge> edges = new PageDataIterable<>(link -> edgeService.findEdgesByTenantId(tenantId, link), 1024); |
||||
|
long currentTimeMillis = System.currentTimeMillis(); |
||||
|
|
||||
|
for (Edge edge : edges) { |
||||
|
Optional<AttributeKvEntry> attributeOpt = attributesService.find(tenantId, edge.getId(), AttributeScope.SERVER_SCOPE, DefaultDeviceStateService.LAST_CONNECT_TIME).get(); |
||||
|
if (attributeOpt.isPresent()) { |
||||
|
Optional<Long> lastConnectTimeOpt = attributeOpt.get().getLongValue(); |
||||
|
if (lastConnectTimeOpt.isPresent() && isTopicExpired(lastConnectTimeOpt.get(), currentTimeMillis)) { |
||||
|
String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edge.getId()).getTopic(); |
||||
|
TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); |
||||
|
if (kafkaAdmin.isTopicEmpty(topic)) { |
||||
|
kafkaAdmin.deleteTopic(topic); |
||||
|
log.info("Removed outdated topic for tenant {} and edge {} older than {}", tenantId, edge.getName(), Date.from(Instant.ofEpochMilli(currentTimeMillis - ONE_MONTH_MILLIS))); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private boolean isTopicExpired(long lastConnectTime, long currentTimeMillis) { |
||||
|
return lastConnectTime + ONE_MONTH_MILLIS < currentTimeMillis; |
||||
|
} |
||||
|
|
||||
|
} |
||||
Loading…
Reference in new issue