|
|
|
@ -28,6 +28,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica |
|
|
|
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; |
|
|
|
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; |
|
|
|
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; |
|
|
|
import org.thingsboard.server.queue.TbQueueAdmin; |
|
|
|
import org.thingsboard.server.queue.TbQueueConsumer; |
|
|
|
import org.thingsboard.server.queue.TbQueueProducer; |
|
|
|
import org.thingsboard.server.queue.TbQueueRequestTemplate; |
|
|
|
@ -36,10 +37,11 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; |
|
|
|
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|
|
|
import org.thingsboard.server.queue.discovery.PartitionService; |
|
|
|
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; |
|
|
|
import org.thingsboard.server.queue.kafka.TBKafkaAdmin; |
|
|
|
import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; |
|
|
|
import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; |
|
|
|
import org.thingsboard.server.queue.kafka.TbKafkaAdmin; |
|
|
|
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; |
|
|
|
import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; |
|
|
|
import org.thingsboard.server.queue.kafka.TbKafkaSettings; |
|
|
|
import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; |
|
|
|
import org.thingsboard.server.queue.settings.TbQueueCoreSettings; |
|
|
|
import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; |
|
|
|
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; |
|
|
|
@ -62,13 +64,20 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi |
|
|
|
private final TbQueueTransportNotificationSettings transportNotificationSettings; |
|
|
|
private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; |
|
|
|
|
|
|
|
private final TbQueueAdmin coreAdmin; |
|
|
|
private final TbQueueAdmin ruleEngineAdmin; |
|
|
|
private final TbQueueAdmin jsExecutorAdmin; |
|
|
|
private final TbQueueAdmin transportApiAdmin; |
|
|
|
private final TbQueueAdmin notificationAdmin; |
|
|
|
|
|
|
|
public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, |
|
|
|
TbServiceInfoProvider serviceInfoProvider, |
|
|
|
TbQueueCoreSettings coreSettings, |
|
|
|
TbQueueRuleEngineSettings ruleEngineSettings, |
|
|
|
TbQueueTransportApiSettings transportApiSettings, |
|
|
|
TbQueueTransportNotificationSettings transportNotificationSettings, |
|
|
|
TbQueueRemoteJsInvokeSettings jsInvokeSettings) { |
|
|
|
TbQueueRemoteJsInvokeSettings jsInvokeSettings, |
|
|
|
TbKafkaTopicConfigs kafkaTopicConfigs) { |
|
|
|
this.partitionService = partitionService; |
|
|
|
this.kafkaSettings = kafkaSettings; |
|
|
|
this.serviceInfoProvider = serviceInfoProvider; |
|
|
|
@ -77,143 +86,160 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi |
|
|
|
this.transportApiSettings = transportApiSettings; |
|
|
|
this.transportNotificationSettings = transportNotificationSettings; |
|
|
|
this.jsInvokeSettings = jsInvokeSettings; |
|
|
|
|
|
|
|
this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); |
|
|
|
this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); |
|
|
|
this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); |
|
|
|
this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); |
|
|
|
this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() { |
|
|
|
TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToTransportMsg>> requestBuilder = TBKafkaProducerTemplate.builder(); |
|
|
|
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToTransportMsg>> requestBuilder = TbKafkaProducerTemplate.builder(); |
|
|
|
requestBuilder.settings(kafkaSettings); |
|
|
|
requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); |
|
|
|
requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); |
|
|
|
requestBuilder.admin(notificationAdmin); |
|
|
|
return requestBuilder.build(); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() { |
|
|
|
TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToRuleEngineMsg>> requestBuilder = TBKafkaProducerTemplate.builder(); |
|
|
|
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToRuleEngineMsg>> requestBuilder = TbKafkaProducerTemplate.builder(); |
|
|
|
requestBuilder.settings(kafkaSettings); |
|
|
|
requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); |
|
|
|
requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); |
|
|
|
requestBuilder.admin(ruleEngineAdmin); |
|
|
|
return requestBuilder.build(); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { |
|
|
|
TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> requestBuilder = TBKafkaProducerTemplate.builder(); |
|
|
|
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> requestBuilder = TbKafkaProducerTemplate.builder(); |
|
|
|
requestBuilder.settings(kafkaSettings); |
|
|
|
requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); |
|
|
|
requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); |
|
|
|
requestBuilder.admin(ruleEngineAdmin); |
|
|
|
return requestBuilder.build(); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { |
|
|
|
TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToCoreMsg>> requestBuilder = TBKafkaProducerTemplate.builder(); |
|
|
|
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToCoreMsg>> requestBuilder = TbKafkaProducerTemplate.builder(); |
|
|
|
requestBuilder.settings(kafkaSettings); |
|
|
|
requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); |
|
|
|
requestBuilder.defaultTopic(coreSettings.getTopic()); |
|
|
|
requestBuilder.admin(coreAdmin); |
|
|
|
return requestBuilder.build(); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { |
|
|
|
TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToCoreNotificationMsg>> requestBuilder = TBKafkaProducerTemplate.builder(); |
|
|
|
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToCoreNotificationMsg>> requestBuilder = TbKafkaProducerTemplate.builder(); |
|
|
|
requestBuilder.settings(kafkaSettings); |
|
|
|
requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); |
|
|
|
requestBuilder.defaultTopic(coreSettings.getTopic()); |
|
|
|
requestBuilder.admin(coreAdmin); |
|
|
|
return requestBuilder.build(); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { |
|
|
|
String queueName = configuration.getName(); |
|
|
|
TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToRuleEngineMsg>> consumerBuilder = TBKafkaConsumerTemplate.builder(); |
|
|
|
TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToRuleEngineMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder(); |
|
|
|
consumerBuilder.settings(kafkaSettings); |
|
|
|
consumerBuilder.topic(ruleEngineSettings.getTopic()); |
|
|
|
consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); |
|
|
|
consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); |
|
|
|
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); |
|
|
|
consumerBuilder.admin(ruleEngineAdmin); |
|
|
|
return consumerBuilder.build(); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { |
|
|
|
TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> consumerBuilder = TBKafkaConsumerTemplate.builder(); |
|
|
|
TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder(); |
|
|
|
consumerBuilder.settings(kafkaSettings); |
|
|
|
consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); |
|
|
|
consumerBuilder.clientId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); |
|
|
|
consumerBuilder.groupId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); |
|
|
|
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); |
|
|
|
consumerBuilder.admin(notificationAdmin); |
|
|
|
return consumerBuilder.build(); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() { |
|
|
|
TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToCoreMsg>> consumerBuilder = TBKafkaConsumerTemplate.builder(); |
|
|
|
TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToCoreMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder(); |
|
|
|
consumerBuilder.settings(kafkaSettings); |
|
|
|
consumerBuilder.topic(coreSettings.getTopic()); |
|
|
|
consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); |
|
|
|
consumerBuilder.groupId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); |
|
|
|
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); |
|
|
|
consumerBuilder.admin(coreAdmin); |
|
|
|
return consumerBuilder.build(); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { |
|
|
|
TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToCoreNotificationMsg>> consumerBuilder = TBKafkaConsumerTemplate.builder(); |
|
|
|
TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToCoreNotificationMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder(); |
|
|
|
consumerBuilder.settings(kafkaSettings); |
|
|
|
consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); |
|
|
|
consumerBuilder.clientId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); |
|
|
|
consumerBuilder.groupId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); |
|
|
|
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); |
|
|
|
consumerBuilder.admin(notificationAdmin); |
|
|
|
return consumerBuilder.build(); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() { |
|
|
|
TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder<TbProtoQueueMsg<TransportApiRequestMsg>> consumerBuilder = TBKafkaConsumerTemplate.builder(); |
|
|
|
TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<TransportApiRequestMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder(); |
|
|
|
consumerBuilder.settings(kafkaSettings); |
|
|
|
consumerBuilder.topic(transportApiSettings.getRequestsTopic()); |
|
|
|
consumerBuilder.clientId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); |
|
|
|
consumerBuilder.groupId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); |
|
|
|
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); |
|
|
|
consumerBuilder.admin(transportApiAdmin); |
|
|
|
return consumerBuilder.build(); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() { |
|
|
|
TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder<TbProtoQueueMsg<TransportApiResponseMsg>> requestBuilder = TBKafkaProducerTemplate.builder(); |
|
|
|
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<TransportApiResponseMsg>> requestBuilder = TbKafkaProducerTemplate.builder(); |
|
|
|
requestBuilder.settings(kafkaSettings); |
|
|
|
requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); |
|
|
|
requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); |
|
|
|
requestBuilder.admin(transportApiAdmin); |
|
|
|
return requestBuilder.build(); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
@Bean |
|
|
|
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { |
|
|
|
TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>> requestBuilder = TBKafkaProducerTemplate.builder(); |
|
|
|
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>> requestBuilder = TbKafkaProducerTemplate.builder(); |
|
|
|
requestBuilder.settings(kafkaSettings); |
|
|
|
requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); |
|
|
|
requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); |
|
|
|
requestBuilder.admin(jsExecutorAdmin); |
|
|
|
|
|
|
|
TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> responseBuilder = TBKafkaConsumerTemplate.builder(); |
|
|
|
TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> responseBuilder = TbKafkaConsumerTemplate.builder(); |
|
|
|
responseBuilder.settings(kafkaSettings); |
|
|
|
responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); |
|
|
|
responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); |
|
|
|
responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); |
|
|
|
// responseBuilder.autoCommit(true);
|
|
|
|
// responseBuilder.autoCommitIntervalMs(autoCommitInterval);
|
|
|
|
responseBuilder.decoder(msg -> { |
|
|
|
JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); |
|
|
|
JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); |
|
|
|
return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); |
|
|
|
} |
|
|
|
); |
|
|
|
responseBuilder.admin(jsExecutorAdmin); |
|
|
|
|
|
|
|
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder |
|
|
|
<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder(); |
|
|
|
builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); |
|
|
|
builder.queueAdmin(jsExecutorAdmin); |
|
|
|
builder.requestTemplate(requestBuilder.build()); |
|
|
|
builder.responseTemplate(responseBuilder.build()); |
|
|
|
builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); |
|
|
|
|