YevhenBondarenko
6 years ago
23 changed files with 438 additions and 459 deletions
@ -0,0 +1,53 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.queue.common; |
|||
|
|||
import com.google.common.util.concurrent.ListeningExecutorService; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.queue.TbQueueMsg; |
|||
|
|||
import java.util.concurrent.Executors; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
@Slf4j |
|||
public abstract class AbstractParallelTbQueueConsumerTemplate<R, T extends TbQueueMsg> extends AbstractTbQueueConsumerTemplate<R, T> { |
|||
|
|||
protected ListeningExecutorService consumerExecutor; |
|||
|
|||
public AbstractParallelTbQueueConsumerTemplate(String topic) { |
|||
super(topic); |
|||
} |
|||
|
|||
protected void initNewExecutor(int threadPoolSize) { |
|||
if (consumerExecutor != null) { |
|||
consumerExecutor.shutdown(); |
|||
try { |
|||
consumerExecutor.awaitTermination(1, TimeUnit.MINUTES); |
|||
} catch (InterruptedException e) { |
|||
log.trace("Interrupted while waiting for consumer executor to stop"); |
|||
} |
|||
} |
|||
consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(threadPoolSize)); |
|||
} |
|||
|
|||
protected void shutdownExecutor() { |
|||
if (consumerExecutor != null) { |
|||
consumerExecutor.shutdownNow(); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,152 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.queue.common; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.queue.TbQueueConsumer; |
|||
import org.thingsboard.server.queue.TbQueueMsg; |
|||
|
|||
import java.io.IOException; |
|||
import java.util.ArrayList; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.Set; |
|||
import java.util.concurrent.locks.Lock; |
|||
import java.util.concurrent.locks.ReentrantLock; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@Slf4j |
|||
public abstract class AbstractTbQueueConsumerTemplate<R, T extends TbQueueMsg> implements TbQueueConsumer<T> { |
|||
|
|||
private volatile boolean subscribed; |
|||
protected volatile boolean stopped = false; |
|||
protected volatile Set<TopicPartitionInfo> partitions; |
|||
protected final Lock consumerLock = new ReentrantLock(); |
|||
|
|||
@Getter |
|||
private final String topic; |
|||
|
|||
public AbstractTbQueueConsumerTemplate(String topic) { |
|||
this.topic = topic; |
|||
} |
|||
|
|||
@Override |
|||
public void subscribe() { |
|||
consumerLock.lock(); |
|||
try { |
|||
partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); |
|||
subscribed = false; |
|||
} finally { |
|||
consumerLock.unlock(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void subscribe(Set<TopicPartitionInfo> partitions) { |
|||
consumerLock.lock(); |
|||
try { |
|||
this.partitions = partitions; |
|||
subscribed = false; |
|||
} finally { |
|||
consumerLock.unlock(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public List<T> poll(long durationInMillis) { |
|||
if (!subscribed && partitions == null) { |
|||
try { |
|||
Thread.sleep(durationInMillis); |
|||
} catch (InterruptedException e) { |
|||
log.debug("Failed to await subscription", e); |
|||
} |
|||
} else { |
|||
long pollStartTs = System.currentTimeMillis(); |
|||
consumerLock.lock(); |
|||
try { |
|||
if (!subscribed) { |
|||
List<String> topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); |
|||
doSubscribe(topicNames); |
|||
subscribed = true; |
|||
} |
|||
|
|||
List<R> records = doPoll(durationInMillis); |
|||
if (!records.isEmpty()) { |
|||
List<T> result = new ArrayList<>(records.size()); |
|||
records.forEach(record -> { |
|||
try { |
|||
if (record != null) { |
|||
result.add(decode(record)); |
|||
} |
|||
} catch (IOException e) { |
|||
log.error("Failed decode record: [{}]", record); |
|||
throw new RuntimeException("Failed to decode record: ", e); |
|||
} |
|||
}); |
|||
return result; |
|||
} else { |
|||
long pollDuration = System.currentTimeMillis() - pollStartTs; |
|||
if (pollDuration < durationInMillis) { |
|||
try { |
|||
Thread.sleep(durationInMillis - pollDuration); |
|||
} catch (InterruptedException e) { |
|||
if (!stopped) { |
|||
log.error("Failed to wait.", e); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} finally { |
|||
consumerLock.unlock(); |
|||
} |
|||
} |
|||
return Collections.emptyList(); |
|||
} |
|||
|
|||
@Override |
|||
public void commit() { |
|||
consumerLock.lock(); |
|||
try { |
|||
doCommit(); |
|||
} finally { |
|||
consumerLock.unlock(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void unsubscribe() { |
|||
stopped = true; |
|||
consumerLock.lock(); |
|||
try { |
|||
doUnsubscribe(); |
|||
} finally { |
|||
consumerLock.unlock(); |
|||
} |
|||
} |
|||
|
|||
abstract protected List<R> doPoll(long durationInMillis); |
|||
|
|||
abstract protected T decode(R record) throws IOException; |
|||
|
|||
abstract protected void doSubscribe(List<String> topicNames); |
|||
|
|||
abstract protected void doCommit(); |
|||
|
|||
abstract protected void doUnsubscribe(); |
|||
|
|||
} |
|||
Loading…
Reference in new issue