Andrew Shvayka
8 years ago
32 changed files with 1382 additions and 217 deletions
@ -0,0 +1,102 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.script; |
|||
|
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import java.util.Map; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
|
|||
/** |
|||
* Created by ashvayka on 26.09.18. |
|||
*/ |
|||
@Slf4j |
|||
public abstract class AbstractJsInvokeService implements JsInvokeService { |
|||
|
|||
protected Map<UUID, String> scriptIdToNameMap = new ConcurrentHashMap<>(); |
|||
protected Map<UUID, AtomicInteger> blackListedFunctions = new ConcurrentHashMap<>(); |
|||
|
|||
@Override |
|||
public ListenableFuture<UUID> eval(JsScriptType scriptType, String scriptBody, String... argNames) { |
|||
UUID scriptId = UUID.randomUUID(); |
|||
String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_'); |
|||
String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames); |
|||
return doEval(scriptId, functionName, jsScript); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Object> invokeFunction(UUID scriptId, Object... args) { |
|||
String functionName = scriptIdToNameMap.get(scriptId); |
|||
if (functionName == null) { |
|||
return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!")); |
|||
} |
|||
if (!isBlackListed(scriptId)) { |
|||
return doInvokeFunction(scriptId, functionName, args); |
|||
} else { |
|||
return Futures.immediateFailedFuture( |
|||
new RuntimeException("Script is blacklisted due to maximum error count " + getMaxErrors() + "!")); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Void> release(UUID scriptId) { |
|||
String functionName = scriptIdToNameMap.get(scriptId); |
|||
if (functionName != null) { |
|||
try { |
|||
scriptIdToNameMap.remove(scriptId); |
|||
blackListedFunctions.remove(scriptId); |
|||
doRelease(scriptId, functionName); |
|||
} catch (Exception e) { |
|||
return Futures.immediateFailedFuture(e); |
|||
} |
|||
} |
|||
return Futures.immediateFuture(null); |
|||
} |
|||
|
|||
protected abstract ListenableFuture<UUID> doEval(UUID scriptId, String functionName, String scriptBody); |
|||
|
|||
protected abstract ListenableFuture<Object> doInvokeFunction(UUID scriptId, String functionName, Object[] args); |
|||
|
|||
protected abstract void doRelease(UUID scriptId, String functionName) throws Exception; |
|||
|
|||
protected abstract int getMaxErrors(); |
|||
|
|||
protected void onScriptExecutionError(UUID scriptId) { |
|||
blackListedFunctions.computeIfAbsent(scriptId, key -> new AtomicInteger(0)).incrementAndGet(); |
|||
} |
|||
|
|||
private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { |
|||
switch (scriptType) { |
|||
case RULE_NODE_SCRIPT: |
|||
return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); |
|||
default: |
|||
throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); |
|||
} |
|||
} |
|||
|
|||
private boolean isBlackListed(UUID scriptId) { |
|||
if (blackListedFunctions.containsKey(scriptId)) { |
|||
AtomicInteger errorCount = blackListedFunctions.get(scriptId); |
|||
return errorCount.get() >= getMaxErrors(); |
|||
} else { |
|||
return false; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,29 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.script; |
|||
|
|||
import java.util.List; |
|||
|
|||
/** |
|||
* Created by ashvayka on 25.09.18. |
|||
*/ |
|||
public class JsInvokeRequest { |
|||
|
|||
private String scriptId; |
|||
private String scriptBody; |
|||
private List<String> args; |
|||
|
|||
} |
|||
@ -0,0 +1,29 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.script; |
|||
|
|||
import java.util.List; |
|||
|
|||
/** |
|||
* Created by ashvayka on 25.09.18. |
|||
*/ |
|||
public class JsInvokeResponse { |
|||
|
|||
private String scriptId; |
|||
private String scriptBody; |
|||
private List<String> args; |
|||
|
|||
} |
|||
@ -0,0 +1,29 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.script; |
|||
|
|||
import org.thingsboard.server.gen.js.JsInvokeProtos; |
|||
import org.thingsboard.server.kafka.TbKafkaEncoder; |
|||
|
|||
/** |
|||
* Created by ashvayka on 25.09.18. |
|||
*/ |
|||
public class RemoteJsRequestEncoder implements TbKafkaEncoder<JsInvokeProtos.RemoteJsRequest> { |
|||
@Override |
|||
public byte[] encode(JsInvokeProtos.RemoteJsRequest value) { |
|||
return value.toByteArray(); |
|||
} |
|||
} |
|||
@ -0,0 +1,32 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.script; |
|||
|
|||
import org.thingsboard.server.gen.js.JsInvokeProtos; |
|||
import org.thingsboard.server.kafka.TbKafkaDecoder; |
|||
|
|||
import java.io.IOException; |
|||
|
|||
/** |
|||
* Created by ashvayka on 25.09.18. |
|||
*/ |
|||
public class RemoteJsResponseDecoder implements TbKafkaDecoder<JsInvokeProtos.RemoteJsResponse> { |
|||
|
|||
@Override |
|||
public JsInvokeProtos.RemoteJsResponse decode(byte[] data) throws IOException { |
|||
return JsInvokeProtos.RemoteJsResponse.parseFrom(data); |
|||
} |
|||
} |
|||
@ -0,0 +1,95 @@ |
|||
<!-- |
|||
|
|||
Copyright © 2016-2018 The Thingsboard Authors |
|||
|
|||
Licensed under the Apache License, Version 2.0 (the "License"); |
|||
you may not use this file except in compliance with the License. |
|||
You may obtain a copy of the License at |
|||
|
|||
http://www.apache.org/licenses/LICENSE-2.0 |
|||
|
|||
Unless required by applicable law or agreed to in writing, software |
|||
distributed under the License is distributed on an "AS IS" BASIS, |
|||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
See the License for the specific language governing permissions and |
|||
limitations under the License. |
|||
|
|||
--> |
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
|||
<modelVersion>4.0.0</modelVersion> |
|||
<parent> |
|||
<groupId>org.thingsboard</groupId> |
|||
<version>2.1.1-SNAPSHOT</version> |
|||
<artifactId>common</artifactId> |
|||
</parent> |
|||
<groupId>org.thingsboard.common</groupId> |
|||
<artifactId>queue</artifactId> |
|||
<packaging>jar</packaging> |
|||
|
|||
<name>Thingsboard Server Queue components</name> |
|||
<url>https://thingsboard.io</url> |
|||
|
|||
<properties> |
|||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> |
|||
<main.dir>${basedir}/../..</main.dir> |
|||
</properties> |
|||
|
|||
<dependencies> |
|||
<dependency> |
|||
<groupId>org.thingsboard.common</groupId> |
|||
<artifactId>data</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.thingsboard.common</groupId> |
|||
<artifactId>message</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.apache.kafka</groupId> |
|||
<artifactId>kafka-clients</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework</groupId> |
|||
<artifactId>spring-context-support</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-autoconfigure</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>com.google.guava</groupId> |
|||
<artifactId>guava</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>com.google.code.gson</groupId> |
|||
<artifactId>gson</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.slf4j</groupId> |
|||
<artifactId>slf4j-api</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.slf4j</groupId> |
|||
<artifactId>log4j-over-slf4j</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>ch.qos.logback</groupId> |
|||
<artifactId>logback-core</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>ch.qos.logback</groupId> |
|||
<artifactId>logback-classic</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>junit</groupId> |
|||
<artifactId>junit</artifactId> |
|||
<scope>test</scope> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.mockito</groupId> |
|||
<artifactId>mockito-all</artifactId> |
|||
<scope>test</scope> |
|||
</dependency> |
|||
</dependencies> |
|||
|
|||
</project> |
|||
@ -0,0 +1,49 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.kafka; |
|||
|
|||
import org.apache.kafka.clients.admin.AdminClient; |
|||
import org.apache.kafka.clients.admin.CreateTopicsResult; |
|||
import org.apache.kafka.clients.admin.NewTopic; |
|||
import org.apache.kafka.clients.consumer.ConsumerRecords; |
|||
import org.apache.kafka.clients.consumer.KafkaConsumer; |
|||
|
|||
import java.time.Duration; |
|||
import java.util.Collections; |
|||
import java.util.Properties; |
|||
|
|||
/** |
|||
* Created by ashvayka on 24.09.18. |
|||
*/ |
|||
public class TBKafkaAdmin { |
|||
|
|||
AdminClient client; |
|||
|
|||
public TBKafkaAdmin() { |
|||
Properties props = new Properties(); |
|||
props.put("bootstrap.servers", "localhost:9092"); |
|||
props.put("group.id", "test"); |
|||
props.put("enable.auto.commit", "true"); |
|||
props.put("auto.commit.interval.ms", "1000"); |
|||
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); |
|||
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); |
|||
client = AdminClient.create(props); |
|||
} |
|||
|
|||
public CreateTopicsResult createTopic(NewTopic topic){ |
|||
return client.createTopics(Collections.singletonList(topic)); |
|||
} |
|||
} |
|||
@ -0,0 +1,71 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.kafka; |
|||
|
|||
import lombok.Builder; |
|||
import lombok.Getter; |
|||
import org.apache.kafka.clients.consumer.ConsumerConfig; |
|||
import org.apache.kafka.clients.consumer.ConsumerRecord; |
|||
import org.apache.kafka.clients.consumer.ConsumerRecords; |
|||
import org.apache.kafka.clients.consumer.KafkaConsumer; |
|||
|
|||
import java.io.IOException; |
|||
import java.time.Duration; |
|||
import java.util.Collections; |
|||
import java.util.Properties; |
|||
|
|||
/** |
|||
* Created by ashvayka on 24.09.18. |
|||
*/ |
|||
public class TBKafkaConsumerTemplate<T> { |
|||
|
|||
private final KafkaConsumer<String, byte[]> consumer; |
|||
private final TbKafkaDecoder<T> decoder; |
|||
@Getter |
|||
private final String topic; |
|||
|
|||
@Builder |
|||
private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder<T> decoder, |
|||
String clientId, String groupId, String topic, |
|||
boolean autoCommit, long autoCommitIntervalMs) { |
|||
Properties props = settings.toProps(); |
|||
props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); |
|||
props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); |
|||
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit); |
|||
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); |
|||
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); |
|||
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); |
|||
this.consumer = new KafkaConsumer<>(props); |
|||
this.decoder = decoder; |
|||
this.topic = topic; |
|||
} |
|||
|
|||
public void subscribe() { |
|||
consumer.subscribe(Collections.singletonList(topic)); |
|||
} |
|||
|
|||
public void unsubscribe() { |
|||
consumer.unsubscribe(); |
|||
} |
|||
|
|||
public ConsumerRecords<String, byte[]> poll(Duration duration) { |
|||
return consumer.poll(duration); |
|||
} |
|||
|
|||
public T decode(ConsumerRecord<String, byte[]> record) throws IOException { |
|||
return decoder.decode(record.value()); |
|||
} |
|||
} |
|||
@ -0,0 +1,83 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.kafka; |
|||
|
|||
import lombok.Builder; |
|||
import lombok.Getter; |
|||
import org.apache.kafka.clients.producer.KafkaProducer; |
|||
import org.apache.kafka.clients.producer.ProducerConfig; |
|||
import org.apache.kafka.clients.producer.ProducerRecord; |
|||
import org.apache.kafka.clients.producer.RecordMetadata; |
|||
import org.apache.kafka.common.PartitionInfo; |
|||
import org.apache.kafka.common.header.Header; |
|||
|
|||
import java.util.List; |
|||
import java.util.Properties; |
|||
import java.util.concurrent.Future; |
|||
|
|||
/** |
|||
* Created by ashvayka on 24.09.18. |
|||
*/ |
|||
public class TBKafkaProducerTemplate<T> { |
|||
|
|||
private final KafkaProducer<String, byte[]> producer; |
|||
private final TbKafkaEncoder<T> encoder; |
|||
private final TbKafkaPartitioner<T> partitioner; |
|||
private final List<PartitionInfo> partitionInfoList; |
|||
@Getter |
|||
private final String defaultTopic; |
|||
|
|||
@Builder |
|||
private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaEncoder<T> encoder, TbKafkaPartitioner<T> partitioner, String defaultTopic) { |
|||
Properties props = settings.toProps(); |
|||
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); |
|||
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); |
|||
this.producer = new KafkaProducer<>(props); |
|||
//Maybe this should not be cached, but we don't plan to change size of partitions
|
|||
this.partitionInfoList = producer.partitionsFor(defaultTopic); |
|||
this.encoder = encoder; |
|||
this.partitioner = partitioner; |
|||
this.defaultTopic = defaultTopic; |
|||
} |
|||
|
|||
public Future<RecordMetadata> send(String key, T value) { |
|||
return send(key, value, null, null); |
|||
} |
|||
|
|||
public Future<RecordMetadata> send(String key, T value, Iterable<Header> headers) { |
|||
return send(key, value, null, headers); |
|||
} |
|||
|
|||
public Future<RecordMetadata> send(String key, T value, Long timestamp, Iterable<Header> headers) { |
|||
return send(this.defaultTopic, key, value, timestamp, headers); |
|||
} |
|||
|
|||
public Future<RecordMetadata> send(String topic, String key, T value, Long timestamp, Iterable<Header> headers) { |
|||
byte[] data = encoder.encode(value); |
|||
ProducerRecord<String, byte[]> record; |
|||
Integer partition = getPartition(topic, key, value, data); |
|||
record = new ProducerRecord<>(this.defaultTopic, partition, timestamp, key, data, headers); |
|||
return producer.send(record); |
|||
} |
|||
|
|||
private Integer getPartition(String topic, String key, T value, byte[] data) { |
|||
if (partitioner == null) { |
|||
return null; |
|||
} else { |
|||
return partitioner.partition(this.defaultTopic, key, value, data, partitionInfoList); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,68 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.kafka; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.kafka.clients.consumer.ConsumerRecords; |
|||
import org.apache.kafka.clients.producer.ProducerRecord; |
|||
import org.apache.kafka.common.header.Header; |
|||
|
|||
import java.nio.charset.StandardCharsets; |
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.concurrent.atomic.LongAdder; |
|||
|
|||
/** |
|||
* Created by ashvayka on 24.09.18. |
|||
*/ |
|||
@Slf4j |
|||
public class TbJsEvaluator { |
|||
|
|||
// public static void main(String[] args) {
|
|||
// ExecutorService executorService = Executors.newCachedThreadPool();
|
|||
//
|
|||
// TBKafkaConsumerTemplate requestConsumer = new TBKafkaConsumerTemplate();
|
|||
// requestConsumer.subscribe("requests");
|
|||
//
|
|||
// LongAdder responseCounter = new LongAdder();
|
|||
// TBKafkaProducerTemplate responseProducer = new TBKafkaProducerTemplate();
|
|||
// executorService.submit((Runnable) () -> {
|
|||
// while (true) {
|
|||
// ConsumerRecords<String, String> requests = requestConsumer.poll(100);
|
|||
// requests.forEach(request -> {
|
|||
// Header header = request.headers().lastHeader("responseTopic");
|
|||
// ProducerRecord<String, String> response = new ProducerRecord<>(new String(header.value(), StandardCharsets.UTF_8),
|
|||
// request.key(), request.value());
|
|||
// responseProducer.send(response);
|
|||
// responseCounter.add(1);
|
|||
// });
|
|||
// }
|
|||
// });
|
|||
//
|
|||
// executorService.submit((Runnable) () -> {
|
|||
// while (true) {
|
|||
// log.warn("Requests: [{}], Responses: [{}]", responseCounter.longValue(), responseCounter.longValue());
|
|||
// try {
|
|||
// Thread.sleep(1000L);
|
|||
// } catch (InterruptedException e) {
|
|||
// e.printStackTrace();
|
|||
// }
|
|||
// }
|
|||
// });
|
|||
//
|
|||
// }
|
|||
|
|||
} |
|||
@ -0,0 +1,27 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.kafka; |
|||
|
|||
import java.io.IOException; |
|||
|
|||
/** |
|||
* Created by ashvayka on 25.09.18. |
|||
*/ |
|||
public interface TbKafkaDecoder<T> { |
|||
|
|||
T decode(byte[] data) throws IOException; |
|||
|
|||
} |
|||
@ -0,0 +1,25 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.kafka; |
|||
|
|||
/** |
|||
* Created by ashvayka on 25.09.18. |
|||
*/ |
|||
public interface TbKafkaEncoder<T> { |
|||
|
|||
byte[] encode(T value); |
|||
|
|||
} |
|||
@ -0,0 +1,30 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.kafka; |
|||
|
|||
import org.apache.kafka.clients.producer.Partitioner; |
|||
import org.apache.kafka.common.PartitionInfo; |
|||
|
|||
import java.util.List; |
|||
|
|||
/** |
|||
* Created by ashvayka on 25.09.18. |
|||
*/ |
|||
public interface TbKafkaPartitioner<T> extends Partitioner { |
|||
|
|||
int partition(String topic, String key, T value, byte[] encodedValue, List<PartitionInfo> partitions); |
|||
|
|||
} |
|||
@ -0,0 +1,32 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.kafka; |
|||
|
|||
import lombok.Data; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
|||
import org.springframework.stereotype.Component; |
|||
|
|||
/** |
|||
* Created by ashvayka on 25.09.18. |
|||
*/ |
|||
@Data |
|||
public class TbKafkaProperty { |
|||
|
|||
private String key; |
|||
private String value; |
|||
} |
|||
@ -0,0 +1,173 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* <p> |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* <p> |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.kafka; |
|||
|
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.SettableFuture; |
|||
import lombok.Builder; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.kafka.clients.admin.CreateTopicsResult; |
|||
import org.apache.kafka.clients.admin.NewTopic; |
|||
import org.apache.kafka.clients.consumer.ConsumerRecords; |
|||
import org.apache.kafka.common.header.Header; |
|||
import org.apache.kafka.common.header.internals.RecordHeader; |
|||
|
|||
import java.io.IOException; |
|||
import java.nio.ByteBuffer; |
|||
import java.nio.charset.StandardCharsets; |
|||
import java.time.Duration; |
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ConcurrentMap; |
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.concurrent.TimeoutException; |
|||
|
|||
/** |
|||
* Created by ashvayka on 25.09.18. |
|||
*/ |
|||
@Slf4j |
|||
public class TbKafkaRequestTemplate<Request, Response> { |
|||
|
|||
private final TBKafkaProducerTemplate<Request> requestTemplate; |
|||
private final TBKafkaConsumerTemplate<Response> responseTemplate; |
|||
private final ConcurrentMap<UUID, ResponseMetaData<Response>> pendingRequests; |
|||
private final ExecutorService executor; |
|||
private final long maxRequestTimeout; |
|||
private final long maxPendingRequests; |
|||
private final long pollInterval; |
|||
private volatile long tickTs = 0L; |
|||
private volatile long tickSize = 0L; |
|||
private volatile boolean stopped = false; |
|||
|
|||
@Builder |
|||
public TbKafkaRequestTemplate(TBKafkaProducerTemplate<Request> requestTemplate, TBKafkaConsumerTemplate<Response> responseTemplate, |
|||
long maxRequestTimeout, |
|||
long maxPendingRequests, |
|||
long pollInterval, |
|||
ExecutorService executor) { |
|||
this.requestTemplate = requestTemplate; |
|||
this.responseTemplate = responseTemplate; |
|||
this.pendingRequests = new ConcurrentHashMap<>(); |
|||
this.maxRequestTimeout = maxRequestTimeout; |
|||
this.maxPendingRequests = maxPendingRequests; |
|||
this.pollInterval = pollInterval; |
|||
if (executor != null) { |
|||
this.executor = executor; |
|||
} else { |
|||
this.executor = Executors.newSingleThreadExecutor(); |
|||
} |
|||
} |
|||
|
|||
public void init() { |
|||
try { |
|||
TBKafkaAdmin admin = new TBKafkaAdmin(); |
|||
CreateTopicsResult result = admin.createTopic(new NewTopic(responseTemplate.getTopic(), 1, (short) 1)); |
|||
result.all().get(); |
|||
} catch (Exception e) { |
|||
log.trace("Failed to create topic: {}", e.getMessage(), e); |
|||
} |
|||
tickTs = System.currentTimeMillis(); |
|||
responseTemplate.subscribe(); |
|||
executor.submit(() -> { |
|||
long nextCleanupMs = 0L; |
|||
while (!stopped) { |
|||
ConsumerRecords<String, byte[]> responses = responseTemplate.poll(Duration.ofMillis(pollInterval)); |
|||
responses.forEach(response -> { |
|||
Header requestIdHeader = response.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); |
|||
if (requestIdHeader == null) { |
|||
log.error("[{}] Missing requestIdHeader", response); |
|||
} |
|||
UUID requestId = bytesToUuid(requestIdHeader.value()); |
|||
ResponseMetaData<Response> expectedResponse = pendingRequests.remove(requestId); |
|||
if (expectedResponse == null) { |
|||
log.trace("[{}] Invalid or stale request", requestId); |
|||
} else { |
|||
try { |
|||
expectedResponse.future.set(responseTemplate.decode(response)); |
|||
} catch (IOException e) { |
|||
expectedResponse.future.setException(e); |
|||
} |
|||
} |
|||
}); |
|||
tickTs = System.currentTimeMillis(); |
|||
tickSize = pendingRequests.size(); |
|||
if (nextCleanupMs < tickTs) { |
|||
//cleanup;
|
|||
pendingRequests.entrySet().forEach(kv -> { |
|||
if (kv.getValue().expTime < tickTs) { |
|||
ResponseMetaData<Response> staleRequest = pendingRequests.remove(kv.getKey()); |
|||
if (staleRequest != null) { |
|||
staleRequest.future.setException(new TimeoutException()); |
|||
} |
|||
} |
|||
}); |
|||
nextCleanupMs = tickTs + maxRequestTimeout; |
|||
} |
|||
} |
|||
}); |
|||
} |
|||
|
|||
public void stop() { |
|||
stopped = true; |
|||
} |
|||
|
|||
public ListenableFuture<Response> post(String key, Request request) { |
|||
if (tickSize > maxPendingRequests) { |
|||
return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); |
|||
} |
|||
UUID requestId = UUID.randomUUID(); |
|||
List<Header> headers = new ArrayList<>(2); |
|||
headers.add(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))); |
|||
headers.add(new RecordHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic()))); |
|||
SettableFuture<Response> future = SettableFuture.create(); |
|||
pendingRequests.putIfAbsent(requestId, new ResponseMetaData<>(tickTs + maxRequestTimeout, future)); |
|||
requestTemplate.send(key, request, headers); |
|||
return future; |
|||
} |
|||
|
|||
private byte[] uuidToBytes(UUID uuid) { |
|||
ByteBuffer buf = ByteBuffer.allocate(16); |
|||
buf.putLong(uuid.getMostSignificantBits()); |
|||
buf.putLong(uuid.getLeastSignificantBits()); |
|||
return buf.array(); |
|||
} |
|||
|
|||
private static UUID bytesToUuid(byte[] bytes) { |
|||
ByteBuffer bb = ByteBuffer.wrap(bytes); |
|||
long firstLong = bb.getLong(); |
|||
long secondLong = bb.getLong(); |
|||
return new UUID(firstLong, secondLong); |
|||
} |
|||
|
|||
private byte[] stringToBytes(String string) { |
|||
return string.getBytes(StandardCharsets.UTF_8); |
|||
} |
|||
|
|||
private static class ResponseMetaData<T> { |
|||
private final long expTime; |
|||
private final SettableFuture<T> future; |
|||
|
|||
ResponseMetaData(long ts, SettableFuture<T> future) { |
|||
this.expTime = ts; |
|||
this.future = future; |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,73 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.kafka; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.kafka.clients.producer.ProducerConfig; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
|||
import org.springframework.stereotype.Component; |
|||
|
|||
import java.util.List; |
|||
import java.util.Properties; |
|||
|
|||
/** |
|||
* Created by ashvayka on 25.09.18. |
|||
*/ |
|||
@Slf4j |
|||
@ConditionalOnProperty(prefix = "kafka", value = "enabled", havingValue = "true", matchIfMissing = false) |
|||
@Component |
|||
public class TbKafkaSettings { |
|||
|
|||
public static final String REQUEST_ID_HEADER = "requestId"; |
|||
public static final String RESPONSE_TOPIC_HEADER = "responseTopic"; |
|||
|
|||
|
|||
@Value("${kafka.bootstrap.server}") |
|||
private String servers; |
|||
|
|||
@Value("${kafka.acks}") |
|||
private String acks; |
|||
|
|||
@Value("${kafka.retries}") |
|||
private int retries; |
|||
|
|||
@Value("${kafka.batch.size}") |
|||
private long batchSize; |
|||
|
|||
@Value("${kafka.linger.ms}") |
|||
private long lingerMs; |
|||
|
|||
@Value("${kafka.buffer.memory}") |
|||
private long bufferMemory; |
|||
|
|||
@Value("${kafka.other:null}") |
|||
private List<TbKafkaProperty> other; |
|||
|
|||
public Properties toProps() { |
|||
Properties props = new Properties(); |
|||
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, servers); |
|||
props.put(ProducerConfig.ACKS_CONFIG, acks); |
|||
props.put(ProducerConfig.RETRIES_CONFIG, retries); |
|||
props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize); |
|||
props.put(ProducerConfig.LINGER_MS_CONFIG, lingerMs); |
|||
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory); |
|||
if(other != null){ |
|||
other.forEach(kv -> props.put(kv.getKey(), kv.getValue())); |
|||
} |
|||
return props; |
|||
} |
|||
} |
|||
@ -0,0 +1,114 @@ |
|||
/** |
|||
* Copyright © 2016-2018 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.kafka; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.kafka.clients.admin.CreateTopicsResult; |
|||
import org.apache.kafka.clients.admin.NewTopic; |
|||
import org.apache.kafka.clients.consumer.ConsumerRecords; |
|||
import org.apache.kafka.clients.producer.ProducerRecord; |
|||
import org.apache.kafka.common.header.Header; |
|||
import org.apache.kafka.common.header.internals.RecordHeader; |
|||
|
|||
import java.nio.charset.StandardCharsets; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ConcurrentMap; |
|||
import java.util.concurrent.ExecutionException; |
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.concurrent.atomic.LongAdder; |
|||
|
|||
/** |
|||
* Created by ashvayka on 24.09.18. |
|||
*/ |
|||
@Slf4j |
|||
public class TbRuleEngineEmulator { |
|||
//
|
|||
// public static void main(String[] args) throws InterruptedException, ExecutionException {
|
|||
// ConcurrentMap<String, String> pendingRequestsMap = new ConcurrentHashMap<>();
|
|||
//
|
|||
// ExecutorService executorService = Executors.newCachedThreadPool();
|
|||
//
|
|||
// String responseTopic = "server" + Math.abs((int) (5000.0 * Math.random()));
|
|||
// try {
|
|||
// TBKafkaAdmin admin = new TBKafkaAdmin();
|
|||
// CreateTopicsResult result = admin.createTopic(new NewTopic(responseTopic, 1, (short) 1));
|
|||
// result.all().get();
|
|||
// } catch (Exception e) {
|
|||
// log.warn("Failed to create topic: {}", e.getMessage(), e);
|
|||
// }
|
|||
//
|
|||
// List<Header> headers = Collections.singletonList(new RecordHeader("responseTopic", responseTopic.getBytes(StandardCharsets.UTF_8)));
|
|||
//
|
|||
// TBKafkaConsumerTemplate responseConsumer = new TBKafkaConsumerTemplate();
|
|||
// TBKafkaProducerTemplate requestProducer = new TBKafkaProducerTemplate();
|
|||
//
|
|||
// LongAdder requestCounter = new LongAdder();
|
|||
// LongAdder responseCounter = new LongAdder();
|
|||
//
|
|||
// responseConsumer.subscribe(responseTopic);
|
|||
// executorService.submit((Runnable) () -> {
|
|||
// while (true) {
|
|||
// ConsumerRecords<String, String> responses = responseConsumer.poll(100);
|
|||
// responses.forEach(response -> {
|
|||
// String expectedResponse = pendingRequestsMap.remove(response.key());
|
|||
// if (expectedResponse == null) {
|
|||
// log.error("[{}] Invalid request", response.key());
|
|||
// } else if (!expectedResponse.equals(response.value())) {
|
|||
// log.error("[{}] Invalid response: {} instead of {}", response.key(), response.value(), expectedResponse);
|
|||
// }
|
|||
// responseCounter.add(1);
|
|||
// });
|
|||
// }
|
|||
// });
|
|||
//
|
|||
// executorService.submit((Runnable) () -> {
|
|||
// int i = 0;
|
|||
// while (true) {
|
|||
// String requestId = UUID.randomUUID().toString();
|
|||
// String expectedResponse = UUID.randomUUID().toString();
|
|||
// pendingRequestsMap.put(requestId, expectedResponse);
|
|||
// requestProducer.send(new ProducerRecord<>("requests", null, requestId, expectedResponse, headers));
|
|||
// requestCounter.add(1);
|
|||
// i++;
|
|||
// if (i % 10000 == 0) {
|
|||
// try {
|
|||
// Thread.sleep(500L);
|
|||
// } catch (InterruptedException e) {
|
|||
// e.printStackTrace();
|
|||
// }
|
|||
// }
|
|||
// }
|
|||
// });
|
|||
//
|
|||
// executorService.submit((Runnable) () -> {
|
|||
// while (true) {
|
|||
// log.warn("Requests: [{}], Responses: [{}]", requestCounter.longValue(), responseCounter.longValue());
|
|||
// try {
|
|||
// Thread.sleep(1000L);
|
|||
// } catch (InterruptedException e) {
|
|||
// e.printStackTrace();
|
|||
// }
|
|||
// }
|
|||
// });
|
|||
//
|
|||
// Thread.sleep(60000);
|
|||
// }
|
|||
|
|||
} |
|||
@ -0,0 +1,35 @@ |
|||
<?xml version="1.0" encoding="UTF-8" ?> |
|||
<!-- |
|||
|
|||
Copyright © 2016-2018 The Thingsboard Authors |
|||
|
|||
Licensed under the Apache License, Version 2.0 (the "License"); |
|||
you may not use this file except in compliance with the License. |
|||
You may obtain a copy of the License at |
|||
|
|||
http://www.apache.org/licenses/LICENSE-2.0 |
|||
|
|||
Unless required by applicable law or agreed to in writing, software |
|||
distributed under the License is distributed on an "AS IS" BASIS, |
|||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
See the License for the specific language governing permissions and |
|||
limitations under the License. |
|||
|
|||
--> |
|||
<!DOCTYPE configuration> |
|||
<configuration scan="true" scanPeriod="10 seconds"> |
|||
|
|||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
|||
<encoder> |
|||
<pattern>%d{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern> |
|||
</encoder> |
|||
</appender> |
|||
|
|||
<logger name="org.thingsboard.server" level="INFO" /> |
|||
<logger name="akka" level="INFO" /> |
|||
|
|||
<root level="INFO"> |
|||
<appender-ref ref="STDOUT"/> |
|||
</root> |
|||
|
|||
</configuration> |
|||
Loading…
Reference in new issue