ViacheslavKlimov
1 year ago
23 changed files with 350 additions and 98 deletions
@ -0,0 +1,164 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.queue.common; |
|||
|
|||
import lombok.Builder; |
|||
import lombok.Getter; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.common.util.ThingsBoardExecutors; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.common.stats.MessagesStats; |
|||
import org.thingsboard.server.queue.TbQueueConsumer; |
|||
import org.thingsboard.server.queue.TbQueueHandler; |
|||
import org.thingsboard.server.queue.TbQueueMsg; |
|||
import org.thingsboard.server.queue.TbQueueProducer; |
|||
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
|||
|
|||
import java.util.List; |
|||
import java.util.Set; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.ScheduledExecutorService; |
|||
import java.util.concurrent.TimeoutException; |
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
import java.util.function.Function; |
|||
|
|||
@Slf4j |
|||
public class PartitionedQueueResponseTemplate<Request extends TbQueueMsg, Response extends TbQueueMsg> extends AbstractTbQueueTemplate { |
|||
|
|||
@Getter |
|||
private final PartitionedQueueConsumerManager<Request> requestConsumer; |
|||
private final TbQueueProducer<Response> responseProducer; |
|||
|
|||
private final TbQueueHandler<Request, Response> handler; |
|||
private final long pollInterval; |
|||
private final int maxPendingRequests; |
|||
private final long requestTimeout; |
|||
private final MessagesStats stats; |
|||
|
|||
private final ScheduledExecutorService scheduler; |
|||
private final ExecutorService callbackExecutor; |
|||
|
|||
private final AtomicInteger pendingRequestCount = new AtomicInteger(); |
|||
|
|||
@Builder |
|||
public PartitionedQueueResponseTemplate(String key, |
|||
TbQueueHandler<Request, Response> handler, |
|||
String requestsTopic, |
|||
Function<TopicPartitionInfo, TbQueueConsumer<Request>> consumerCreator, |
|||
TbQueueProducer<Response> responseProducer, |
|||
long pollInterval, |
|||
long requestTimeout, |
|||
int maxPendingRequests, |
|||
ExecutorService consumerExecutor, |
|||
ExecutorService callbackExecutor, |
|||
ExecutorService consumerTaskExecutor, |
|||
MessagesStats stats) { |
|||
this.scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor(key + "-queue-response-template-scheduler"); |
|||
this.callbackExecutor = callbackExecutor; |
|||
this.handler = handler; |
|||
this.requestConsumer = PartitionedQueueConsumerManager.<Request>create() |
|||
.queueKey(key + "-requests") |
|||
.topic(requestsTopic) |
|||
.pollInterval(pollInterval) |
|||
.msgPackProcessor((requests, consumer, config) -> processRequests(requests, consumer)) |
|||
.consumerCreator((config, tpi) -> consumerCreator.apply(tpi)) |
|||
.consumerExecutor(consumerExecutor) |
|||
.scheduler(scheduler) |
|||
.taskExecutor(consumerTaskExecutor) |
|||
.build(); |
|||
this.responseProducer = responseProducer; |
|||
this.pollInterval = pollInterval; |
|||
this.maxPendingRequests = maxPendingRequests; |
|||
this.requestTimeout = requestTimeout; |
|||
this.stats = stats; |
|||
} |
|||
|
|||
private void processRequests(List<Request> requests, TbQueueConsumer<Request> consumer) { |
|||
while (pendingRequestCount.get() >= maxPendingRequests) { |
|||
try { |
|||
Thread.sleep(pollInterval); |
|||
} catch (InterruptedException e) { |
|||
log.trace("Failed to wait until the server has capacity to handle new requests", e); |
|||
} |
|||
} |
|||
|
|||
requests.forEach(request -> { |
|||
long currentTime = System.currentTimeMillis(); |
|||
long expireTs = bytesToLong(request.getHeaders().get(EXPIRE_TS_HEADER)); |
|||
if (expireTs >= currentTime) { |
|||
byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); |
|||
if (requestIdHeader == null) { |
|||
log.error("[{}] Missing requestId in header", request); |
|||
return; |
|||
} |
|||
byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); |
|||
if (responseTopicHeader == null) { |
|||
log.error("[{}] Missing response topic in header", request); |
|||
return; |
|||
} |
|||
UUID requestId = bytesToUuid(requestIdHeader); |
|||
String responseTopic = bytesToString(responseTopicHeader); |
|||
try { |
|||
pendingRequestCount.getAndIncrement(); |
|||
stats.incrementTotal(); |
|||
AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), |
|||
response -> { |
|||
pendingRequestCount.decrementAndGet(); |
|||
response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); |
|||
responseProducer.send(TopicPartitionInfo.builder().topic(responseTopic).build(), response, null); |
|||
stats.incrementSuccessful(); |
|||
}, |
|||
e -> { |
|||
pendingRequestCount.decrementAndGet(); |
|||
if (e.getCause() != null && e.getCause() instanceof TimeoutException) { |
|||
log.warn("[{}] Timeout to process the request: {}", requestId, request, e); |
|||
} else { |
|||
log.trace("[{}] Failed to process the request: {}", requestId, request, e); |
|||
} |
|||
stats.incrementFailed(); |
|||
}, |
|||
requestTimeout, |
|||
scheduler, |
|||
callbackExecutor); |
|||
} catch (Throwable e) { |
|||
pendingRequestCount.decrementAndGet(); |
|||
log.warn("[{}] Failed to process the request: {}", requestId, request, e); |
|||
stats.incrementFailed(); |
|||
} |
|||
} |
|||
}); |
|||
consumer.commit(); |
|||
} |
|||
|
|||
public void subscribe(Set<TopicPartitionInfo> partitions) { |
|||
requestConsumer.update(partitions); |
|||
} |
|||
|
|||
public void stop() { |
|||
if (requestConsumer != null) { |
|||
requestConsumer.stop(); |
|||
requestConsumer.awaitStop(); |
|||
} |
|||
if (responseProducer != null) { |
|||
responseProducer.stop(); |
|||
} |
|||
if (scheduler != null) { |
|||
scheduler.shutdownNow(); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,70 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.queue.edqs; |
|||
|
|||
import com.google.common.util.concurrent.ListeningExecutorService; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import jakarta.annotation.PostConstruct; |
|||
import jakarta.annotation.PreDestroy; |
|||
import lombok.Getter; |
|||
import lombok.RequiredArgsConstructor; |
|||
import org.springframework.context.annotation.Lazy; |
|||
import org.springframework.stereotype.Component; |
|||
import org.thingsboard.common.util.ThingsBoardExecutors; |
|||
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
|||
|
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.concurrent.ScheduledExecutorService; |
|||
|
|||
@Lazy |
|||
@Component |
|||
@Getter |
|||
@RequiredArgsConstructor |
|||
public class EdqsExecutors { |
|||
|
|||
private final EdqsConfig edqsConfig; |
|||
|
|||
private ExecutorService consumersExecutor; |
|||
private ExecutorService consumerTaskExecutor; |
|||
private ScheduledExecutorService scheduler; |
|||
private ListeningExecutorService requestExecutor; |
|||
|
|||
@PostConstruct |
|||
private void init() { |
|||
consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); |
|||
consumerTaskExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-task-executor"); |
|||
scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler"); |
|||
requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(edqsConfig.getRequestExecutorSize(), "edqs-requests")); |
|||
} |
|||
|
|||
@PreDestroy |
|||
private void destroy() { |
|||
if (consumersExecutor != null) { |
|||
consumersExecutor.shutdownNow(); |
|||
} |
|||
if (consumerTaskExecutor != null) { |
|||
consumerTaskExecutor.shutdownNow(); |
|||
} |
|||
if (scheduler != null) { |
|||
scheduler.shutdownNow(); |
|||
} |
|||
if (requestExecutor != null) { |
|||
requestExecutor.shutdownNow(); |
|||
} |
|||
} |
|||
|
|||
} |
|||
Loading…
Reference in new issue