From 23c017567bf59022bef311c9b2d641fc3c31c7a4 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 16 Mar 2020 19:24:56 +0200 Subject: [PATCH] Implementation of Queue Interfaces --- .../server/actors/ActorSystemContext.java | 21 +- .../server/actors/app/AppActor.java | 14 +- .../device/DeviceActorMessageProcessor.java | 2 +- .../actors/rpc/BasicRpcSessionListener.java | 83 ----- .../server/actors/rpc/RpcBroadcastMsg.java | 27 -- .../server/actors/rpc/RpcManagerActor.java | 230 ------------ .../server/actors/rpc/RpcSessionActor.java | 135 ------- .../actors/rpc/RpcSessionClosedMsg.java | 29 -- .../actors/rpc/RpcSessionConnectedMsg.java | 31 -- .../rpc/RpcSessionCreateRequestMsg.java | 35 -- .../actors/rpc/RpcSessionDisconnectedMsg.java | 29 -- .../server/actors/rpc/RpcSessionTellMsg.java | 27 -- .../server/actors/rpc/SessionActorInfo.java | 30 -- .../actors/ruleChain/DefaultTbContext.java | 21 +- .../RuleChainActorMessageProcessor.java | 15 +- .../server/actors/service/ActorService.java | 4 +- .../actors/service/DefaultActorService.java | 213 +++++------ .../server/actors/stats/StatsActor.java | 3 +- .../CurrentServerInstanceService.java | 55 --- .../cluster/discovery/DiscoveryService.java | 33 -- .../discovery/DiscoveryServiceListener.java | 28 -- .../discovery/DummyDiscoveryService.java | 67 ---- .../cluster/discovery/ServerInstance.java | 47 --- .../discovery/ServerInstanceService.java | 24 -- .../cluster/discovery/ZkDiscoveryService.java | 330 ------------------ .../routing/ClusterRoutingService.java | 35 -- .../ConsistentClusterRoutingService.java | 153 -------- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../DefaultTbRuleEngineConsumerService.java | 40 ++- .../queue/TbRuleEngineConsumerStats.java | 59 +--- .../service/rpc/DefaultDeviceRpcService.java | 29 +- .../state/DefaultDeviceStateService.java | 91 ++--- .../DefaultTelemetrySubscriptionService.java | 38 +- .../BaseRuleChainTransactionService.java | 25 +- .../DefaultTbCoreToTransportService.java | 9 +- .../RemoteRuleEngineTransportService.java | 237 ------------- .../transport/RemoteTransportApiService.java | 3 +- .../transport/TransportApiRequestDecoder.java | 31 -- .../TransportApiResponseEncoder.java | 30 -- .../src/main/resources/thingsboard.yml | 3 +- .../ConsistentClusterRoutingServiceTest.java | 71 ++-- .../thingsboard/server/TbQueueProducer.java | 10 +- .../common/DefaultTbQueueRequestTemplate.java | 5 +- .../DefaultTbQueueResponseTemplate.java | 3 +- .../discovery}/ConsistentHashCircle.java | 18 +- .../ConsistentHashPartitionService.java | 221 ++++++++++++ .../DefaultTbServiceInfoProvider.java | 11 +- .../discovery/DummyDiscoveryService.java | 32 ++ .../discovery/PartitionDiscoveryService.java | 9 - .../server/discovery/PartitionService.java | 16 + .../server/discovery/TopicPartitionInfo.java | 23 +- .../ZkPartitionDiscoveryService.java | 122 ++++--- .../environment/EnvironmentLogService.java | 4 +- .../server/kafka/TBKafkaProducerTemplate.java | 50 +-- .../memory/InMemoryTbQueueProducer.java | 17 +- ...ava => InMemoryMonolithQueueProvider.java} | 12 +- .../provider/KafkaMonolithQueueProvider.java | 119 +++++++ .../provider/KafkaTbCoreQueueProvider.java | 33 +- .../KafkaTbRuleEngineQueueProvider.java | 86 +++++ .../provider/KafkaTransportQueueProvider.java | 8 +- .../provider/TbRuleEngineQueueProvider.java | 61 ++++ common/transport/transport-api/pom.xml | 4 + .../service/DefaultTransportService.java | 38 +- 63 files changed, 1055 insertions(+), 2236 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java rename {application/src/main/java/org/thingsboard/server/service/cluster/routing => common/queue/src/main/java/org/thingsboard/server/discovery}/ConsistentHashCircle.java (66%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java rename {application/src/main/java/org/thingsboard/server/service => common/queue/src/main/java/org/thingsboard/server}/environment/EnvironmentLogService.java (90%) rename common/queue/src/main/java/org/thingsboard/server/provider/{InMemoryTbCoreQueueProvider.java => InMemoryMonolithQueueProvider.java} (85%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index d10436075b..e5a0e6a25a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -65,8 +65,6 @@ import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.kafka.TbNodeIdProvider; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; @@ -108,19 +106,11 @@ public class ActorSystemContext { @Setter private ActorService actorService; - @Autowired - @Getter - private DiscoveryService discoveryService; - @Autowired @Getter @Setter private ComponentDiscoveryService componentService; - @Autowired - @Getter - private ClusterRoutingService routingService; - @Autowired @Getter private DataDecodingEncodingService encodingService; @@ -368,7 +358,8 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.ERROR); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); + //TODO 2.5 +// event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); persistEvent(event); } @@ -377,7 +368,8 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); + //TODO 2.5 +// event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -406,8 +398,11 @@ public class ActorSystemContext { return mapper.createObjectNode().put("server", server.toString()).put("method", method).put("error", body); } + public String getServerAddress() { - return discoveryService.getCurrentServer().getServerAddress().toString(); + //TODO 2.5 +// return discoveryService.getCurrentServer().getServerAddress().toString(); + return null; } public void persistDebugInput(TenantId tenantId, EntityId entityId, TbMsg tbMsg, String relationType) { diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 778c945a32..06f936a9e6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -134,13 +134,15 @@ public class AppActor extends RuleChainManagerActor { } private void onPossibleClusterMsg(SendToClusterMsg msg) { - Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); - if (address.isPresent()) { - systemContext.getRpcService().tell( - systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); - } else { + //TODO 2.5 +// Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); +// if (address.isPresent()) { + +// systemContext.getRpcService().tell( +// systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); +// } else { self().tell(msg.getMsg(), ActorRef.noSender()); - } +// } } private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 6c0dde4044..6b878ef7d8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -260,7 +260,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - //TODO: 2.5 move this as a notification to the queue; + //TODO 2.5 move this as a notification to the queue; private void reportLogicalDeviceActivity() { systemContext.getDeviceStateService().onDeviceActivity(deviceId); } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java b/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java deleted file mode 100644 index 8b502b973c..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; -import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class BasicRpcSessionListener implements GrpcSessionListener { - - private final ClusterRpcCallbackExecutorService callbackExecutorService; - private final ActorService service; - private final ActorRef manager; - private final ActorRef self; - - BasicRpcSessionListener(ActorSystemContext context, ActorRef manager, ActorRef self) { - this.service = context.getActorService(); - this.callbackExecutorService = context.getClusterRpcCallbackExecutor(); - this.manager = manager; - this.self = self; - } - - @Override - public void onConnected(GrpcSession session) { - log.info("[{}][{}] session started", session.getRemoteServer(), getType(session)); - if (!session.isClient()) { - manager.tell(new RpcSessionConnectedMsg(session.getRemoteServer(), session.getSessionId()), self); - } - } - - @Override - public void onDisconnected(GrpcSession session) { - log.info("[{}][{}] session closed", session.getRemoteServer(), getType(session)); - manager.tell(new RpcSessionDisconnectedMsg(session.isClient(), session.getRemoteServer()), self); - } - - @Override - public void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage) { - log.trace("Received session actor msg from [{}][{}]: {}", session.getRemoteServer(), getType(session), clusterMessage); - callbackExecutorService.execute(() -> { - try { - service.onReceivedMsg(session.getRemoteServer(), clusterMessage); - } catch (Exception e) { - log.debug("[{}][{}] Failed to process cluster message: {}", session.getRemoteServer(), getType(session), clusterMessage, e); - } - }); - } - - @Override - public void onError(GrpcSession session, Throwable t) { - log.warn("[{}][{}] session got error -> {}", session.getRemoteServer(), getType(session), t); - manager.tell(new RpcSessionClosedMsg(session.isClient(), session.getRemoteServer()), self); - session.close(); - } - - private static String getType(GrpcSession session) { - return session.isClient() ? "Client" : "Server"; - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java deleted file mode 100644 index f0ff8ce9e1..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcBroadcastMsg { - private final ClusterAPIProtos.ClusterMessage msg; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java deleted file mode 100644 index 133e7375d4..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java +++ /dev/null @@ -1,230 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import akka.actor.OneForOneStrategy; -import akka.actor.Props; -import akka.actor.SupervisorStrategy; -import akka.event.Logging; -import akka.event.LoggingAdapter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import scala.concurrent.duration.Duration; - -import java.util.*; - -/** - * @author Andrew Shvayka - */ -public class RpcManagerActor extends ContextAwareActor { - - private final Map sessionActors; - private final Map> pendingMsgs; - private final ServerAddress instance; - - private RpcManagerActor(ActorSystemContext systemContext) { - super(systemContext); - this.sessionActors = new HashMap<>(); - this.pendingMsgs = new HashMap<>(); - this.instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - - systemContext.getDiscoveryService().getOtherServers().stream() - .filter(otherServer -> otherServer.getServerAddress().compareTo(instance) > 0) - .forEach(otherServer -> onCreateSessionRequest( - new RpcSessionCreateRequestMsg(UUID.randomUUID(), otherServer.getServerAddress(), null))); - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - onMsg((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcBroadcastMsg) { - onMsg((RpcBroadcastMsg) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - onCreateSessionRequest((RpcSessionCreateRequestMsg) msg); - } else if (msg instanceof RpcSessionConnectedMsg) { - onSessionConnected((RpcSessionConnectedMsg) msg); - } else if (msg instanceof RpcSessionDisconnectedMsg) { - onSessionDisconnected((RpcSessionDisconnectedMsg) msg); - } else if (msg instanceof RpcSessionClosedMsg) { - onSessionClosed((RpcSessionClosedMsg) msg); - } else if (msg instanceof ClusterEventMsg) { - onClusterEvent((ClusterEventMsg) msg); - } - } - - private void onMsg(RpcBroadcastMsg msg) { - log.debug("Forwarding msg to session actors {}", msg); - sessionActors.keySet().forEach(address -> { - ClusterAPIProtos.ClusterMessage msgWithServerAddress = msg.getMsg() - .toBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(address.getHost()) - .setPort(address.getPort()) - .build()) - .build(); - onMsg(msgWithServerAddress); - }); - pendingMsgs.values().forEach(queue -> queue.add(msg.getMsg())); - } - - private void onMsg(ClusterAPIProtos.ClusterMessage msg) { - if (msg.hasServerAddress()) { - ServerAddress address = new ServerAddress(msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE); - SessionActorInfo session = sessionActors.get(address); - if (session != null) { - log.debug("{} Forwarding msg to session actor: {}", address, msg); - session.getActor().tell(msg, ActorRef.noSender()); - } else { - log.debug("{} Storing msg to pending queue: {}", address, msg); - Queue queue = pendingMsgs.get(address); - if (queue == null) { - queue = new LinkedList<>(); - pendingMsgs.put(new ServerAddress( - msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE), queue); - } - queue.add(msg); - } - } else { - log.warn("Cluster msg doesn't have server address [{}]", msg); - } - } - - @Override - public void postStop() { - sessionActors.clear(); - pendingMsgs.clear(); - } - - private void onClusterEvent(ClusterEventMsg msg) { - ServerAddress server = msg.getServerAddress(); - if (server.compareTo(instance) > 0) { - if (msg.isAdded()) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(UUID.randomUUID(), server, null)); - } else { - onSessionClose(false, server); - } - } - } - - private void onSessionConnected(RpcSessionConnectedMsg msg) { - register(msg.getRemoteAddress(), msg.getId(), context().sender()); - } - - private void onSessionDisconnected(RpcSessionDisconnectedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private void onSessionClosed(RpcSessionClosedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private boolean isRegistered(ServerAddress address) { - for (ServerInstance server : systemContext.getDiscoveryService().getOtherServers()) { - if (server.getServerAddress().equals(address)) { - return true; - } - } - return false; - } - - private void onSessionClose(boolean reconnect, ServerAddress remoteAddress) { - log.info("[{}] session closed. Should reconnect: {}", remoteAddress, reconnect); - SessionActorInfo sessionRef = sessionActors.get(remoteAddress); - if (sessionRef != null && context().sender() != null && context().sender().equals(sessionRef.actor)) { - context().stop(sessionRef.actor); - sessionActors.remove(remoteAddress); - pendingMsgs.remove(remoteAddress); - if (reconnect) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(sessionRef.sessionId, remoteAddress, null)); - } - } - } - - private void onCreateSessionRequest(RpcSessionCreateRequestMsg msg) { - if (msg.getRemoteAddress() != null) { - if (!sessionActors.containsKey(msg.getRemoteAddress())) { - ActorRef actorRef = createSessionActor(msg); - register(msg.getRemoteAddress(), msg.getMsgUid(), actorRef); - } - } else { - createSessionActor(msg); - } - } - - private void register(ServerAddress remoteAddress, UUID uuid, ActorRef sender) { - sessionActors.put(remoteAddress, new SessionActorInfo(uuid, sender)); - log.info("[{}][{}] Registering session actor.", remoteAddress, uuid); - Queue data = pendingMsgs.remove(remoteAddress); - if (data != null) { - log.info("[{}][{}] Forwarding {} pending messages.", remoteAddress, uuid, data.size()); - data.forEach(msg -> sender.tell(new RpcSessionTellMsg(msg), ActorRef.noSender())); - } else { - log.info("[{}][{}] No pending messages to forward.", remoteAddress, uuid); - } - } - - private ActorRef createSessionActor(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Creating session actor.", msg.getMsgUid()); - ActorRef actor = context().actorOf( - Props.create(new RpcSessionActor.ActorCreator(systemContext, msg.getMsgUid())) - .withDispatcher(DefaultActorService.RPC_DISPATCHER_NAME)); - actor.tell(msg, context().self()); - return actor; - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - public ActorCreator(ActorSystemContext context) { - super(context); - } - - @Override - public RpcManagerActor create() { - return new RpcManagerActor(context); - } - } - - @Override - public SupervisorStrategy supervisorStrategy() { - return strategy; - } - - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { - log.warn("Unknown failure", t); - return SupervisorStrategy.resume(); - }); -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java deleted file mode 100644 index af2a7f0631..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; - -import java.util.UUID; - -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class RpcSessionActor extends ContextAwareActor { - - - private final UUID sessionId; - private GrpcSession session; - private GrpcSessionListener listener; - - private RpcSessionActor(ActorSystemContext systemContext, UUID sessionId) { - super(systemContext); - this.sessionId = sessionId; - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - tell((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - initSession((RpcSessionCreateRequestMsg) msg); - } - } - - private void tell(ClusterAPIProtos.ClusterMessage msg) { - if (session != null) { - session.sendMsg(msg); - } else { - log.trace("Failed to send message due to missing session!"); - } - } - - @Override - public void postStop() { - if (session != null) { - log.info("Closing session -> {}", session.getRemoteServer()); - try { - session.close(); - } catch (RuntimeException e) { - log.trace("Failed to close session!", e); - } - } - } - - private void initSession(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Initializing session", context().self()); - ServerAddress remoteServer = msg.getRemoteAddress(); - listener = new BasicRpcSessionListener(systemContext, context().parent(), context().self()); - if (msg.getRemoteAddress() == null) { - // Server session - session = new GrpcSession(listener); - session.setOutputStream(msg.getResponseObserver()); - session.initInputStream(); - session.initOutputStream(); - systemContext.getRpcService().onSessionCreated(msg.getMsgUid(), session.getInputStream()); - } else { - // Client session - ManagedChannel channel = ManagedChannelBuilder.forAddress(remoteServer.getHost(), remoteServer.getPort()).usePlaintext().build(); - session = new GrpcSession(remoteServer, listener, channel); - session.initInputStream(); - - ClusterRpcServiceGrpc.ClusterRpcServiceStub stub = ClusterRpcServiceGrpc.newStub(channel); - StreamObserver outputStream = stub.handleMsgs(session.getInputStream()); - - session.setOutputStream(outputStream); - session.initOutputStream(); - outputStream.onNext(toConnectMsg()); - } - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - private final UUID sessionId; - - public ActorCreator(ActorSystemContext context, UUID sessionId) { - super(context); - this.sessionId = sessionId; - } - - @Override - public RpcSessionActor create() { - return new RpcSessionActor(context, sessionId); - } - } - - private ClusterAPIProtos.ClusterMessage toConnectMsg() { - ServerAddress instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - return ClusterAPIProtos.ClusterMessage.newBuilder().setMessageType(CONNECT_RPC_MESSAGE).setServerAddress( - ClusterAPIProtos.ServerAddress.newBuilder().setHost(instance.getHost()) - .setPort(instance.getPort()).build()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java deleted file mode 100644 index bb87d2ea88..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionClosedMsg { - - private final boolean client; - private final ServerAddress remoteAddress; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java deleted file mode 100644 index 489a5bfa17..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionConnectedMsg { - - private final ServerAddress remoteAddress; - private final UUID id; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java deleted file mode 100644 index df41a7e95f..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import io.grpc.stub.StreamObserver; -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionCreateRequestMsg { - - private final UUID msgUid; - private final ServerAddress remoteAddress; - private final StreamObserver responseObserver; - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java deleted file mode 100644 index 14e7504637..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionDisconnectedMsg { - - private final boolean client; - private final ServerAddress remoteAddress; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java deleted file mode 100644 index 3832d6eb94..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionTellMsg { - private final ClusterAPIProtos.ClusterMessage msg; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java b/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java deleted file mode 100644 index 811713819a..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import lombok.Data; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -public final class SessionActorInfo { - protected final UUID sessionId; - protected final ActorRef actor; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index b700007498..f5ea5c399f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -121,8 +121,10 @@ class DefaultTbContext implements TbContext { @Override public boolean isLocalEntity(EntityId entityId) { - Optional address = mainCtx.getRoutingService().resolveById(entityId); - return !address.isPresent(); + //TODO 2.5 +// Optional address = mainCtx.getRoutingService().resolveById(entityId); +// return !address.isPresent(); + return true; } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -353,13 +355,14 @@ class DefaultTbContext implements TbContext { src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); mainCtx.getDeviceRpcService().forwardServerSideRPCRequestToDeviceActor(request, response -> { if (src.isRestApiCall()) { - ServerAddress requestOriginAddress; - if (!StringUtils.isEmpty(src.getOriginHost())) { - requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); - } else { - requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); - } - mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); + //TODO 2.5 +// ServerAddress requestOriginAddress; +// if (!StringUtils.isEmpty(src.getOriginHost())) { +// requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); +// } else { +// requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); +// } +// mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); } consumer.accept(RuleEngineDeviceRpcResponse.builder() .deviceId(src.getDeviceId()) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 49d019c1dc..095af7f62d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -230,20 +230,21 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor address = systemContext.getRoutingService().resolveById(originatorEntityId); - - if (address.isPresent()) { - onRemoteTellNext(address.get(), envelope); - } else { + //TODO 2.5 +// Optional address = systemContext.getRoutingService().resolveById(originatorEntityId); +// if (address.isPresent()) { +// onRemoteTellNext(address.get(), envelope); +// } else { onLocalTellNext(envelope); - } +// } } private void onRemoteTellNext(ServerAddress serverAddress, RuleNodeToRuleChainTellNextMsg envelope) { TbMsg msg = envelope.getMsg(); log.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator()); envelope = new RemoteToRuleChainTellNextMsg(envelope, tenantId, entityId); - systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); + //TODO 2.5 +// systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); } private void onLocalTellNext(RuleNodeToRuleChainTellNextMsg envelope) { diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index 48a31127f4..890128e645 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -21,10 +21,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.SessionMsgProcessor; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.rpc.RpcMsgListener; -public interface ActorService extends SessionMsgProcessor, RpcMsgListener, DiscoveryServiceListener { +public interface ActorService extends SessionMsgProcessor { void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index e5da93b9e2..fc9eb537c1 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -19,7 +19,6 @@ import akka.actor.ActorRef; import akka.actor.ActorSystem; import akka.actor.Props; import akka.actor.Terminated; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -32,25 +31,16 @@ import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.app.AppActor; import org.thingsboard.server.actors.app.AppInitMsg; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcManagerActor; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; import org.thingsboard.server.actors.stats.StatsActor; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.cluster.ToAllNodesMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.state.DeviceStateService; import scala.concurrent.Await; import scala.concurrent.Future; @@ -60,8 +50,6 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.concurrent.atomic.AtomicInteger; -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; - @Service @Slf4j public class DefaultActorService implements ActorService { @@ -77,12 +65,6 @@ public class DefaultActorService implements ActorService { @Autowired private ActorSystemContext actorContext; - @Autowired - private ClusterRpcService rpcService; - - @Autowired - private DiscoveryService discoveryService; - @Autowired private DeviceStateService deviceStateService; @@ -102,13 +84,9 @@ public class DefaultActorService implements ActorService { appActor = system.actorOf(Props.create(new AppActor.ActorCreator(actorContext)).withDispatcher(APP_DISPATCHER_NAME), "appActor"); actorContext.setAppActor(appActor); - rpcManagerActor = system.actorOf(Props.create(new RpcManagerActor.ActorCreator(actorContext)).withDispatcher(CORE_DISPATCHER_NAME), - "rpcManagerActor"); - ActorRef statsActor = system.actorOf(Props.create(new StatsActor.ActorCreator(actorContext)).withDispatcher(CORE_DISPATCHER_NAME), "statsActor"); actorContext.setStatsActor(statsActor); - rpcService.init(this); log.info("Actor system initialized."); } @@ -134,22 +112,23 @@ public class DefaultActorService implements ActorService { appActor.tell(msg, ActorRef.noSender()); } - @Override - public void onServerAdded(ServerInstance server) { - log.trace("Processing onServerAdded msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), true)); - } - - @Override - public void onServerUpdated(ServerInstance server) { - //Do nothing - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.trace("Processing onServerRemoved msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), false)); - } + //TODO 2.5 +// @Override +// public void onServerAdded(ServerInstance server) { +// log.trace("Processing onServerAdded msg: {}", server); +// broadcast(new ClusterEventMsg(server.getServerAddress(), true)); +// } +// +// @Override +// public void onServerUpdated(ServerInstance server) { +// //Do nothing +// } +// +// @Override +// public void onServerRemoved(ServerInstance server) { +// log.trace("Processing onServerRemoved msg: {}", server); +// broadcast(new ClusterEventMsg(server.getServerAddress(), false)); +// } @Override public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { @@ -172,12 +151,13 @@ public class DefaultActorService implements ActorService { public void broadcast(ToAllNodesMsg msg) { actorContext.getEncodingService().encode(msg); - rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage - .newBuilder() - .setPayload(ByteString - .copyFrom(actorContext.getEncodingService().encode(msg))) - .setMessageType(CLUSTER_ACTOR_MESSAGE) - .build())); + //TODO 2.5 +// rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage +// .newBuilder() +// .setPayload(ByteString +// .copyFrom(actorContext.getEncodingService().encode(msg))) +// .setMessageType(CLUSTER_ACTOR_MESSAGE) +// .build())); appActor.tell(msg, ActorRef.noSender()); } @@ -204,79 +184,78 @@ public class DefaultActorService implements ActorService { } } - @Override - public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - receivedClusterMsgs.incrementAndGet(); - } - ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); - if (log.isDebugEnabled()) { - log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); - log.info("MSG: {}", msg); - } - switch (msg.getMessageType()) { - case CLUSTER_ACTOR_MESSAGE: - java.util.Optional decodedMsg = actorContext.getEncodingService() - .decode(msg.getPayload().toByteArray()); - if (decodedMsg.isPresent()) { - appActor.tell(decodedMsg.get(), ActorRef.noSender()); - } else { - log.error("Error during decoding cluster proto message"); - } - break; - case TO_ALL_NODES_MSG: - //TODO - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: - actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: - actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: - actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TRANSACTION_SERVICE_MESSAGE: - actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); - break; - } - } - - @Override - public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onBroadcastMsg(RpcBroadcastMsg msg) { - rpcManagerActor.tell(msg, ActorRef.noSender()); - } + //TODO 2.5 +// @Override +// public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { +// if (statsEnabled) { +// receivedClusterMsgs.incrementAndGet(); +// } +// ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); +// if (log.isDebugEnabled()) { +// log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); +// log.info("MSG: {}", msg); +// } +// switch (msg.getMessageType()) { +// case CLUSTER_ACTOR_MESSAGE: +// java.util.Optional decodedMsg = actorContext.getEncodingService() +// .decode(msg.getPayload().toByteArray()); +// if (decodedMsg.isPresent()) { +// appActor.tell(decodedMsg.get(), ActorRef.noSender()); +// } else { +// log.error("Error during decoding cluster proto message"); +// } +// break; +// case TO_ALL_NODES_MSG: +// //TODO +// break; +// case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: +// actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: +// actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: +// actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: +// actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: +// actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: +// actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: +// actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: +// actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TRANSACTION_SERVICE_MESSAGE: +// actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); +// break; +// } +// } +// @Override +// public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { +// if (statsEnabled) { +// sentClusterMsgs.incrementAndGet(); +// } +// rpcManagerActor.tell(msg, ActorRef.noSender()); +// } +// +// @Override +// public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { +// if (statsEnabled) { +// sentClusterMsgs.incrementAndGet(); +// } +// rpcManagerActor.tell(msg, ActorRef.noSender()); +// } +// @Override +// public void onBroadcastMsg(RpcBroadcastMsg msg) { +// rpcManagerActor.tell(msg, ActorRef.noSender()); +// } @Override public void onDeviceAdded(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 31461a8fac..c8e55bf9ed 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -58,7 +58,8 @@ public class StatsActor extends ContextAwareActor { event.setEntityId(msg.getEntityId()); event.setTenantId(msg.getTenantId()); event.setType(DataConstants.STATS); - event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); + //TODO 2.5 +// event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); systemContext.getEventService().save(event); } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java deleted file mode 100644 index 9a7c83f167..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; - -import javax.annotation.PostConstruct; - -import static org.thingsboard.server.utils.MiscUtils.missingProperty; - -/** - * @author Andrew Shvayka - */ -@Service -@Slf4j -public class CurrentServerInstanceService implements ServerInstanceService { - - @Value("${rpc.bind_host}") - private String rpcHost; - @Value("${rpc.bind_port}") - private Integer rpcPort; - - private ServerInstance self; - - @PostConstruct - public void init() { - Assert.hasLength(rpcHost, missingProperty("rpc.bind_host")); - Assert.notNull(rpcPort, missingProperty("rpc.bind_port")); - self = new ServerInstance(new ServerAddress(rpcHost, rpcPort, ServerType.CORE)); - log.info("Current server instance: [{};{}]", self.getHost(), self.getPort()); - } - - @Override - public ServerInstance getSelf() { - return self; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java deleted file mode 100644 index d0ba27814c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import java.util.List; - -/** - * @author Andrew Shvayka - */ -public interface DiscoveryService { - - void publishCurrentServer(); - - void unpublishCurrentServer(); - - ServerInstance getCurrentServer(); - - List getOtherServers(); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java deleted file mode 100644 index ef6b565817..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -/** - * @author Andrew Shvayka - */ -public interface DiscoveryServiceListener { - - void onServerAdded(ServerInstance server); - - void onServerUpdated(ServerInstance server); - - void onServerRemoved(ServerInstance server); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java deleted file mode 100644 index 009a7801e7..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.DependsOn; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import java.util.Collections; -import java.util.List; - -/** - * @author Andrew Shvayka - */ -@Service -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) -@Slf4j -@DependsOn("environmentLogService") -public class DummyDiscoveryService implements DiscoveryService { - - @Autowired - private ServerInstanceService serverInstance; - - @PostConstruct - public void init() { - log.info("Initializing..."); - } - - @Override - public void publishCurrentServer() { - //Do nothing - } - - @Override - public void unpublishCurrentServer() { - //Do nothing - } - - @Override - public ServerInstance getCurrentServer() { - return serverInstance.getSelf(); - } - - @Override - public List getOtherServers() { - return Collections.emptyList(); - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java deleted file mode 100644 index e9325800af..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@ToString -@EqualsAndHashCode(exclude = {"serverInfo", "serverAddress"}) -public final class ServerInstance implements Comparable { - - @Getter - private final String host; - @Getter - private final int port; - @Getter - private final ServerAddress serverAddress; - - public ServerInstance(ServerAddress serverAddress) { - this.serverAddress = serverAddress; - this.host = serverAddress.getHost(); - this.port = serverAddress.getPort(); - } - - @Override - public int compareTo(ServerInstance o) { - return this.serverAddress.compareTo(o.serverAddress); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java deleted file mode 100644 index 095e10f0b3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -/** - * @author Andrew Shvayka - */ -public interface ServerInstanceService { - - ServerInstance getSelf(); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java deleted file mode 100644 index e1cdd5f83e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java +++ /dev/null @@ -1,330 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.SerializationException; -import org.apache.commons.lang3.SerializationUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.imps.CuratorFrameworkState; -import org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; -import org.apache.curator.framework.state.ConnectionState; -import org.apache.curator.framework.state.ConnectionStateListener; -import org.apache.curator.retry.RetryForever; -import org.apache.curator.utils.CloseableUtils; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.context.annotation.Lazy; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.utils.MiscUtils; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED; - -/** - * @author Andrew Shvayka - */ -@Service -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) -@Slf4j -public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheListener { - - @Value("${zk.url}") - private String zkUrl; - @Value("${zk.retry_interval_ms}") - private Integer zkRetryInterval; - @Value("${zk.connection_timeout_ms}") - private Integer zkConnectionTimeout; - @Value("${zk.session_timeout_ms}") - private Integer zkSessionTimeout; - @Value("${zk.zk_dir}") - private String zkDir; - - private String zkNodesDir; - - @Autowired - private ServerInstanceService serverInstance; - - @Autowired - @Lazy - private TelemetrySubscriptionService tsSubService; - - @Autowired - @Lazy - private DeviceStateService deviceStateService; - - @Autowired - @Lazy - private ActorService actorService; - - @Autowired - @Lazy - private ClusterRoutingService routingService; - - private ExecutorService reconnectExecutorService; - - private CuratorFramework client; - private PathChildrenCache cache; - private String nodePath; - - private volatile boolean stopped = true; - - @PostConstruct - public void init() { - log.info("Initializing..."); - Assert.hasLength(zkUrl, MiscUtils.missingProperty("zk.url")); - Assert.notNull(zkRetryInterval, MiscUtils.missingProperty("zk.retry_interval_ms")); - Assert.notNull(zkConnectionTimeout, MiscUtils.missingProperty("zk.connection_timeout_ms")); - Assert.notNull(zkSessionTimeout, MiscUtils.missingProperty("zk.session_timeout_ms")); - - reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); - - log.info("Initializing discovery service using ZK connect string: {}", zkUrl); - - zkNodesDir = zkDir + "/nodes"; - initZkClient(); - } - - private void initZkClient() { - try { - client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); - client.start(); - client.blockUntilConnected(); - cache = new PathChildrenCache(client, zkNodesDir, true); - cache.getListenable().addListener(this); - cache.start(); - stopped = false; - log.info("ZK client connected"); - } catch (Exception e) { - log.error("Failed to connect to ZK: {}", e.getMessage(), e); - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - throw new RuntimeException(e); - } - } - - private void destroyZkClient() { - stopped = true; - try { - unpublishCurrentServer(); - } catch (Exception e) {} - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - log.info("ZK client disconnected"); - } - - @PreDestroy - public void destroy() { - destroyZkClient(); - reconnectExecutorService.shutdownNow(); - log.info("Stopped discovery service"); - } - - @Override - public synchronized void publishCurrentServer() { - ServerInstance self = this.serverInstance.getSelf(); - if (currentServerExists()) { - log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); - } else { - try { - log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); - nodePath = client.create() - .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); - log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); - client.getConnectionStateListenable().addListener(checkReconnect(self)); - } catch (Exception e) { - log.error("Failed to create ZK node", e); - throw new RuntimeException(e); - } - } - } - - private boolean currentServerExists() { - if (nodePath == null) { - return false; - } - try { - ServerInstance self = this.serverInstance.getSelf(); - ServerAddress registeredServerAdress = null; - registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); - if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { - return true; - } - } catch (KeeperException.NoNodeException e) { - log.info("ZK node does not exist: {}", nodePath); - } catch (Exception e) { - log.error("Couldn't check if ZK node exists", e); - } - return false; - } - - private ConnectionStateListener checkReconnect(ServerInstance self) { - return (client, newState) -> { - log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); - if (newState == ConnectionState.LOST) { - reconnectExecutorService.submit(this::reconnect); - } - }; - } - - private volatile boolean reconnectInProgress = false; - - private synchronized void reconnect() { - if (!reconnectInProgress) { - reconnectInProgress = true; - try { - destroyZkClient(); - initZkClient(); - publishCurrentServer(); - } catch (Exception e) { - log.error("Failed to reconnect to ZK: {}", e.getMessage(), e); - } finally { - reconnectInProgress = false; - } - } - } - - @Override - public void unpublishCurrentServer() { - try { - if (nodePath != null) { - client.delete().forPath(nodePath); - } - } catch (Exception e) { - log.error("Failed to delete ZK node {}", nodePath, e); - throw new RuntimeException(e); - } - } - - @Override - public ServerInstance getCurrentServer() { - return serverInstance.getSelf(); - } - - @Override - public List getOtherServers() { - return cache.getCurrentData().stream() - .filter(cd -> !cd.getPath().equals(nodePath)) - .map(cd -> { - try { - return new ServerInstance((ServerAddress) SerializationUtils.deserialize(cd.getData())); - } catch (NoSuchElementException e) { - log.error("Failed to decode ZK node", e); - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting current ZK node."); - if (stopped) { - log.debug("Ignoring application ready event. Service is stopped."); - return; - } - if (client.getState() != CuratorFrameworkState.STARTED) { - log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); - return; - } - publishCurrentServer(); - getOtherServers().forEach( - server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) - ); - } - - @Override - public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent pathChildrenCacheEvent) throws Exception { - if (stopped) { - log.debug("Ignoring {}. Service is stopped.", pathChildrenCacheEvent); - return; - } - if (client.getState() != CuratorFrameworkState.STARTED) { - log.debug("Ignoring {}, ZK client is not started, ZK client state [{}]", pathChildrenCacheEvent, client.getState()); - return; - } - ChildData data = pathChildrenCacheEvent.getData(); - if (data == null) { - log.debug("Ignoring {} due to empty child data", pathChildrenCacheEvent); - return; - } else if (data.getData() == null) { - log.debug("Ignoring {} due to empty child's data", pathChildrenCacheEvent); - return; - } else if (nodePath != null && nodePath.equals(data.getPath())) { - if (pathChildrenCacheEvent.getType() == CHILD_REMOVED) { - log.info("ZK node for current instance is somehow deleted."); - publishCurrentServer(); - } - log.debug("Ignoring event about current server {}", pathChildrenCacheEvent); - return; - } - ServerInstance instance; - try { - ServerAddress serverAddress = SerializationUtils.deserialize(data.getData()); - instance = new ServerInstance(serverAddress); - } catch (SerializationException e) { - log.error("Failed to decode server instance for node {}", data.getPath(), e); - throw e; - } - log.info("Processing [{}] event for [{}:{}]", pathChildrenCacheEvent.getType(), instance.getHost(), instance.getPort()); - switch (pathChildrenCacheEvent.getType()) { - case CHILD_ADDED: - routingService.onServerAdded(instance); - tsSubService.onClusterUpdate(); - deviceStateService.onClusterUpdate(); - actorService.onServerAdded(instance); - break; - case CHILD_UPDATED: - routingService.onServerUpdated(instance); - actorService.onServerUpdated(instance); - break; - case CHILD_REMOVED: - routingService.onServerRemoved(instance); - tsSubService.onClusterUpdate(); - deviceStateService.onClusterUpdate(); - actorService.onServerRemoved(instance); - break; - default: - break; - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java deleted file mode 100644 index e0ed64fdbd..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; - -import java.util.Optional; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -public interface ClusterRoutingService extends DiscoveryServiceListener { - - ServerAddress getCurrentServer(); - - Optional resolveById(EntityId entityId); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java deleted file mode 100644 index e114a9b6ab..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import com.google.common.hash.HashCode; -import com.google.common.hash.HashFunction; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.utils.MiscUtils; - -import javax.annotation.PostConstruct; -import java.util.Arrays; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -/** - * Cluster service implementation based on consistent hash ring - */ - -@Service -@Slf4j -public class ConsistentClusterRoutingService implements ClusterRoutingService { - - @Autowired - private DiscoveryService discoveryService; - - @Value("${cluster.hash_function_name}") - private String hashFunctionName; - @Value("${cluster.vitrual_nodes_size}") - private Integer virtualNodesSize; - - private ServerInstance currentServer; - - private HashFunction hashFunction; - - private ConsistentHashCircle[] circles; - private ConsistentHashCircle rootCircle; - - @PostConstruct - public void init() { - log.info("Initializing Cluster routing service!"); - this.hashFunction = MiscUtils.forName(hashFunctionName); - this.currentServer = discoveryService.getCurrentServer(); - this.circles = new ConsistentHashCircle[ServerType.values().length]; - for (ServerType serverType : ServerType.values()) { - circles[serverType.ordinal()] = new ConsistentHashCircle(); - } - rootCircle = circles[ServerType.CORE.ordinal()]; - addNode(discoveryService.getCurrentServer()); - for (ServerInstance instance : discoveryService.getOtherServers()) { - addNode(instance); - } - logCircle(); - log.info("Cluster routing service initialized!"); - } - - @Override - public ServerAddress getCurrentServer() { - return discoveryService.getCurrentServer().getServerAddress(); - } - - @Override - public Optional resolveById(EntityId entityId) { - return resolveByUuid(rootCircle, entityId.getId()); - } - - private Optional resolveByUuid(ConsistentHashCircle circle, UUID uuid) { - Assert.notNull(uuid); - if (circle.isEmpty()) { - return Optional.empty(); - } - Long hash = hashFunction.newHasher().putLong(uuid.getMostSignificantBits()) - .putLong(uuid.getLeastSignificantBits()).hash().asLong(); - if (!circle.containsKey(hash)) { - ConcurrentNavigableMap tailMap = - circle.tailMap(hash); - hash = tailMap.isEmpty() ? - circle.firstKey() : tailMap.firstKey(); - } - ServerInstance result = circle.get(hash); - if (!currentServer.equals(result)) { - return Optional.of(result.getServerAddress()); - } else { - return Optional.empty(); - } - } - - @Override - public void onServerAdded(ServerInstance server) { - log.info("On server added event: {}", server); - addNode(server); - logCircle(); - } - - @Override - public void onServerUpdated(ServerInstance server) { - log.debug("Ignoring server onUpdate event: {}", server); - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.info("On server removed event: {}", server); - removeNode(server); - logCircle(); - } - - private void addNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].put(hash(instance, i).asLong(), instance); - } - } - - private void removeNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].remove(hash(instance, i).asLong()); - } - } - - private HashCode hash(ServerInstance instance, int i) { - return hashFunction.newHasher().putString(instance.getHost(), MiscUtils.UTF8).putInt(instance.getPort()).putInt(i).hash(); - } - - private void logCircle() { - log.trace("Consistent Hash Circle Start"); - Arrays.asList(circles).forEach(ConsistentHashCircle::log); - log.trace("Consistent Hash Circle End"); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 0d0e38a1b2..295ce1aa13 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -45,7 +45,7 @@ import java.util.function.Function; import java.util.stream.Collectors; @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") @Slf4j public class DefaultTbCoreConsumerService implements TbCoreConsumerService { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 413d5ee9dd..ef2b44bb6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -29,6 +29,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.provider.TbRuleEngineQueueProvider; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -44,26 +45,26 @@ import java.util.function.Function; import java.util.stream.Collectors; @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine')") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") @Slf4j public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerService { - @Value("${queue.rule-engine.poll_interval}") + @Value("${queue.rule_engine.poll_interval}") private long pollDuration; - @Value("${queue.rule-engine.pack_processing_timeout}") + @Value("${queue.rule_engine.pack_processing_timeout}") private long packProcessingTimeout; - @Value("${queue.rule-engine.stats.enabled:false}") + @Value("${queue.rule_engine.stats.enabled:false}") private boolean statsEnabled; private final ActorSystemContext actorContext; - private final TbQueueConsumer> consumer; + private final TbQueueConsumer> consumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private volatile ExecutorService mainConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbRuleEngineConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext) { - this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); + public DefaultTbRuleEngineConsumerService(TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext) { + this.consumer = tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(); this.actorContext = actorContext; } @@ -78,17 +79,17 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS mainConsumerExecutor.execute(() -> { while (!stopped) { try { - List> msgs = consumer.poll(pollDuration); - ConcurrentMap> ackMap = msgs.stream().collect( + List> msgs = consumer.poll(pollDuration); + ConcurrentMap> ackMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); ackMap.forEach((id, msg) -> { TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); try { - TransportProtos.ToCoreMsg toCoreMsg = msg.getValue(); - log.trace("Forwarding message to rule engine {}", toCoreMsg); - if (toCoreMsg.hasToDeviceActorMsg()) { - forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); + TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); + if (toRuleEngineMsg.hasToRuleEngineMsg()) { + forwardToRuleEngineActor(toRuleEngineMsg.getToRuleEngineMsg(), callback); } else { callback.onSuccess(); } @@ -112,11 +113,12 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS }); } - private void forwardToDeviceActor(TransportProtos.TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + //TODO 2.5 + private void forwardToRuleEngineActor(TransportProtos.TransportToRuleEngineMsg toDeviceActorMsg, TbMsgCallback callback) { +// if (statsEnabled) { +// stats.log(toDeviceActorMsg); +// } +// actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); } @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 75711d20b2..a5e8ba94af 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,60 +24,29 @@ import java.util.concurrent.atomic.AtomicInteger; public class TbRuleEngineConsumerStats { private final AtomicInteger totalCounter = new AtomicInteger(0); - private final AtomicInteger sessionEventCounter = new AtomicInteger(0); -// private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); -// private final AtomicInteger postAttributesCounter = new AtomicInteger(0); - private final AtomicInteger getAttributesCounter = new AtomicInteger(0); - private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); - private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); - private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); -// private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); - private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); - private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); + private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); + private final AtomicInteger postAttributesCounter = new AtomicInteger(0); + private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); - public void log(TransportProtos.TransportToDeviceActorMsg msg) { + public void log(TransportProtos.TransportToRuleEngineMsg msg) { totalCounter.incrementAndGet(); - if (msg.hasSessionEvent()) { - sessionEventCounter.incrementAndGet(); + if (msg.hasPostTelemetry()) { + postTelemetryCounter.incrementAndGet(); } -// if (msg.hasPostTelemetry()) { -// postTelemetryCounter.incrementAndGet(); -// } -// if (msg.hasPostAttributes()) { -// postAttributesCounter.incrementAndGet(); -// } - if (msg.hasGetAttributes()) { - getAttributesCounter.incrementAndGet(); + if (msg.hasPostAttributes()) { + postAttributesCounter.incrementAndGet(); } - if (msg.hasSubscribeToAttributes()) { - subscribeToAttributesCounter.incrementAndGet(); - } - if (msg.hasSubscribeToRPC()) { - subscribeToRPCCounter.incrementAndGet(); - } - if (msg.hasToDeviceRPCCallResponse()) { - toDeviceRPCCallResponseCounter.incrementAndGet(); - } -// if (msg.hasToServerRPCCallRequest()) { -// toServerRPCCallRequestCounter.incrementAndGet(); -// } - if (msg.hasSubscriptionInfo()) { - subscriptionInfoCounter.incrementAndGet(); - } - if (msg.hasClaimDevice()) { - claimDeviceCounter.incrementAndGet(); + if (msg.hasToServerRPCCallRequest()) { + toServerRPCCallRequestCounter.incrementAndGet(); } } public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { - log.info("Transport total [{}] sessionEvents [{}] telemetry [{}] attributes [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] " + - "toServerRpc [{}] subInfo [{}] claimDevice [{}] ", - total, sessionEventCounter.getAndSet(0), postTelemetryCounter.getAndSet(0), - postAttributesCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), - subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), - toServerRPCCallRequestCounter.getAndSet(0), subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); + log.info("Transport total [{}] telemetry [{}] attributes [{}] toServerRpc [{}]", + total, postTelemetryCounter.getAndSet(0), + postAttributesCounter.getAndSet(0), toServerRPCCallRequestCounter.getAndSet(0)); } } } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java index 5d303b54ab..8baecbf37a 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java @@ -42,8 +42,6 @@ import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -64,12 +62,6 @@ public class DefaultDeviceRpcService implements DeviceRpcService { private static final ObjectMapper json = new ObjectMapper(); - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService rpcService; - @Autowired private DeviceService deviceService; @@ -106,7 +98,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService { @Override public void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response) { log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), requestOriginAddress); - if (routingService.getCurrentServer().equals(requestOriginAddress)) { + //TODO 2.5 + if (true) {//routingService.getCurrentServer().equals(requestOriginAddress) UUID requestId = response.getId(); Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); if (consumer != null) { @@ -124,7 +117,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService { } else { builder.setError(-1); } - rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); } } @@ -159,7 +153,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService { } RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), proto.getResponse(), error); - processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); + //TODO 2.5 +// processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); } @Override @@ -172,8 +167,9 @@ public class DefaultDeviceRpcService implements DeviceRpcService { ObjectNode entityNode = json.createObjectNode(); TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("requestUUID", msg.getId().toString()); - metaData.putValue("originHost", routingService.getCurrentServer().getHost()); - metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); + //TODO 2.5 +// metaData.putValue("originHost", routingService.getCurrentServer().getHost()); +// metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); metaData.putValue("oneway", Boolean.toString(msg.isOneway())); @@ -197,9 +193,10 @@ public class DefaultDeviceRpcService implements DeviceRpcService { } private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { - ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); - log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); - forward(msg.getDeviceId(), rpcMsg); + //TODO 2.5 +// ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); +// log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); +// forward(msg.getDeviceId(), rpcMsg); } private void forward(DeviceId deviceId, T msg) { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index f40de180ab..31b712644d 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -57,21 +57,31 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import static org.thingsboard.server.common.data.DataConstants.*; +import static org.thingsboard.server.common.data.DataConstants.ACTIVITY_EVENT; +import static org.thingsboard.server.common.data.DataConstants.CONNECT_EVENT; +import static org.thingsboard.server.common.data.DataConstants.DISCONNECT_EVENT; +import static org.thingsboard.server.common.data.DataConstants.INACTIVITY_EVENT; +import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; /** * Created by ashvayka on 01.05.18. @@ -111,12 +121,6 @@ public class DefaultDeviceStateService implements DeviceStateService { @Autowired private TelemetrySubscriptionService tsSubService; - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService clusterRpcService; - @Value("${state.defaultInactivityTimeoutInSec}") @Getter private long defaultInactivityTimeoutInSec; @@ -235,19 +239,20 @@ public class DefaultDeviceStateService implements DeviceStateService { TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.getNextPageLink(); for (Device device : page.getData()) { - if (!routingService.resolveById(device.getId()).isPresent()) { + //TODO 2.5 +// if (!routingService.resolveById(device.getId()).isPresent()) { if (!deviceStates.containsKey(device.getId())) { fetchFutures.add(fetchDeviceState(device)); } - } else { - Set tenantDeviceSet = tenantDevices.get(tenant.getId()); - if (tenantDeviceSet != null) { - tenantDeviceSet.remove(device.getId()); - } - deviceStates.remove(device.getId()); - deviceLastReportedActivity.remove(device.getId()); - deviceLastSavedActivity.remove(device.getId()); - } +// } else { +// Set tenantDeviceSet = tenantDevices.get(tenant.getId()); +// if (tenantDeviceSet != null) { +// tenantDeviceSet.remove(device.getId()); +// } +// deviceStates.remove(device.getId()); +// deviceLastReportedActivity.remove(device.getId()); +// deviceLastSavedActivity.remove(device.getId()); +// } } try { Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); @@ -268,9 +273,10 @@ public class DefaultDeviceStateService implements DeviceStateService { TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.getNextPageLink(); for (Device device : page.getData()) { - if (!routingService.resolveById(device.getId()).isPresent()) { + //TODO 2.5 +// if (!routingService.resolveById(device.getId()).isPresent()) { fetchFutures.add(fetchDeviceState(device)); - } +// } } try { Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); @@ -356,7 +362,8 @@ public class DefaultDeviceStateService implements DeviceStateService { private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) { DeviceStateData deviceStateData = deviceStates.get(deviceId); if (deviceStateData == null) { - if (!routingService.resolveById(deviceId).isPresent()) { + //TODO 2.5 +// if (!routingService.resolveById(deviceId).isPresent()) { Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); if (device != null) { try { @@ -366,7 +373,7 @@ public class DefaultDeviceStateService implements DeviceStateService { log.debug("[{}] Failed to fetch device state!", deviceId, e); } } - } +// } } return deviceStateData; } @@ -389,8 +396,9 @@ public class DefaultDeviceStateService implements DeviceStateService { } private void onDeviceAddedSync(Device device) { - Optional address = routingService.resolveById(device.getId()); - if (!address.isPresent()) { + //TODO 2.5 +// Optional address = routingService.resolveById(device.getId()); +// if (!address.isPresent()) { Futures.addCallback(fetchDeviceState(device), new FutureCallback() { @Override public void onSuccess(@Nullable DeviceStateData state) { @@ -402,9 +410,9 @@ public class DefaultDeviceStateService implements DeviceStateService { log.warn("Failed to register device to the state service", t); } }); - } else { - sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); - } +// } else { +// sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); +// } } private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, ServerAddress address, boolean added, boolean updated, boolean deleted) { @@ -417,12 +425,14 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); } private void onDeviceUpdatedSync(Device device) { - Optional address = routingService.resolveById(device.getId()); - if (!address.isPresent()) { + //TODO 2.5 +// Optional address = routingService.resolveById(device.getId()); +// if (!address.isPresent()) { DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); if (stateData != null) { TbMsgMetaData md = new TbMsgMetaData(); @@ -430,14 +440,15 @@ public class DefaultDeviceStateService implements DeviceStateService { md.putValue("deviceType", device.getType()); stateData.setMetaData(md); } - } else { - sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); - } +// } else { +// sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); +// } } private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) { - Optional address = routingService.resolveById(deviceId); - if (!address.isPresent()) { + //TODO 2.5 +// Optional address = routingService.resolveById(deviceId); +// if (!address.isPresent()) { deviceStates.remove(deviceId); deviceLastReportedActivity.remove(deviceId); deviceLastSavedActivity.remove(deviceId); @@ -448,9 +459,9 @@ public class DefaultDeviceStateService implements DeviceStateService { tenantDevices.remove(tenantId); } } - } else { - sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); - } +// } else { +// sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); +// } } private ListenableFuture fetchDeviceState(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index d29e5de5b7..04952a3bd1 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -56,8 +56,6 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.sub.Subscription; @@ -102,12 +100,6 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio @Autowired private TimeseriesService tsService; - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService rpcService; - @Autowired private EntityViewService entityViewService; @@ -152,7 +144,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio endTime = entityView.getEndTimeMs(); sub = getUpdatedSubscriptionState(entityId, sub, entityView); } - Optional server = routingService.resolveById(entityId); + //TODO 2.5 + Optional server = Optional.empty();//routingService.resolveById(entityId); Subscription subscription; if (server.isPresent()) { ServerAddress address = server.get(); @@ -340,7 +333,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio while (deviceIterator.hasNext()) { Map.Entry> e = deviceIterator.next(); Set subscriptions = e.getValue(); - Optional newAddressOptional = routingService.resolveById(e.getKey()); + //TODO 2.5 + Optional newAddressOptional = Optional.empty();// routingService.resolveById(e.getKey()); if (newAddressOptional.isPresent()) { newAddressOptional.ifPresent(serverAddress -> checkSubscriptionsNewAddress(serverAddress, subscriptions)); } else { @@ -424,7 +418,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } private void onAttributesUpdate(EntityId entityId, String scope, List attributes) { - Optional serverAddress = routingService.resolveById(entityId); + //TODO 2.5 + Optional serverAddress = Optional.empty();//routingService.resolveById(entityId); if (!serverAddress.isPresent()) { onLocalAttributesUpdate(entityId, scope, attributes); if (entityId.getEntityType() == EntityType.DEVICE && DataConstants.SERVER_SCOPE.equalsIgnoreCase(scope)) { @@ -440,7 +435,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } private void onTimeseriesUpdate(EntityId entityId, List ts) { - Optional serverAddress = routingService.resolveById(entityId); + //TODO 2.5 + Optional serverAddress = Optional.empty();//routingService.resolveById(entityId); if (!serverAddress.isPresent()) { onLocalTimeseriesUpdate(entityId, ts); } else { @@ -632,7 +628,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } sub.getKeyStates().entrySet().forEach(e -> builder.addKeyStates( ClusterAPIProtos.SubscriptionKetStateProto.newBuilder().setKey(e.getKey()).setTs(e.getValue()).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteSubUpdate(ServerAddress address, String sessionId, SubscriptionUpdate update) { @@ -657,7 +654,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio builder.addData(dataBuilder.build()); } ); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteAttributesUpdate(ServerAddress address, EntityId entityId, String scope, List attributes) { @@ -666,7 +664,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio builder.setEntityType(entityId.getEntityType().name()); builder.setScope(scope); attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteTimeseriesUpdate(ServerAddress address, EntityId entityId, List ts) { @@ -674,17 +673,20 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio builder.setEntityId(entityId.getId().toString()); builder.setEntityType(entityId.getEntityType().name()); ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteSessionClose(ServerAddress address, String sessionId) { ClusterAPIProtos.SessionCloseProto proto = ClusterAPIProtos.SessionCloseProto.newBuilder().setSessionId(sessionId).build(); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); } private void tellRemoteSubClose(ServerAddress address, String sessionId, int subscriptionId) { ClusterAPIProtos.SubscriptionCloseProto proto = ClusterAPIProtos.SubscriptionCloseProto.newBuilder().setSessionId(sessionId).setSubscriptionId(subscriptionId).build(); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); } private ClusterAPIProtos.KeyValueProto.Builder toKeyValueProto(long ts, KvEntry attr) { diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java index 1aee532c93..173c293363 100644 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java @@ -24,9 +24,6 @@ import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import javax.annotation.PostConstruct; @@ -50,12 +47,6 @@ import java.util.function.Consumer; @Slf4j public class BaseRuleChainTransactionService implements RuleChainTransactionService { - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService clusterRpcService; - @Autowired private DbCallbackExecutorService callbackExecutor; @@ -110,13 +101,14 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ @Override public void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { - Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); - if (address.isPresent()) { - sendTransactionEventToRemoteServer(msg, address.get()); - executeOnSuccess(onSuccess, msg); - } else { + //TODO 2.5 +// Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); +// if (address.isPresent()) { +// sendTransactionEventToRemoteServer(msg, address.get()); +// executeOnSuccess(onSuccess, msg); +// } else { endLocalTransaction(msg, onSuccess, onFailure); - } +// } } @Override @@ -237,6 +229,7 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ private void sendTransactionEventToRemoteServer(TbMsg msg, ServerAddress address) { log.trace("[{}][{}] Originator is monitored on other server: {}", msg.getTransactionData().getOriginatorId(), msg.getTransactionData().getTransactionId(), address); - clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); + //TODO 2.5 +// clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index fd8ee406c2..f7afd105dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,6 +23,7 @@ import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.provider.TbCoreQueueProvider; @@ -34,17 +35,15 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @Slf4j @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") public class DefaultTbCoreToTransportService implements TbCoreToTransportService { - private final TbCoreQueueProvider tbCoreQueueProvider; private final TbQueueProducer> tbTransportProducer; @Value("${queue.notifications.topic}") private String notificationsTopic; public DefaultTbCoreToTransportService(TbCoreQueueProvider tbCoreQueueProvider) { - this.tbCoreQueueProvider = tbCoreQueueProvider; this.tbTransportProducer = tbCoreQueueProvider.getTransportMsgProducer(); } @@ -60,7 +59,7 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, transportMsg); - tbTransportProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + tbTransportProducer.send(TopicPartitionInfo.builder().topic(topic).build(), queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); } private static class QueueCallbackAdaptor implements TbQueueCallback { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java deleted file mode 100644 index eed2b8fa8b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java +++ /dev/null @@ -1,237 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import akka.actor.ActorRef; -import io.github.bucket4j.Bandwidth; -import io.github.bucket4j.BlockingBucket; -import io.github.bucket4j.Bucket4j; -import io.github.bucket4j.local.LocalBucket; -import io.github.bucket4j.local.LocalBucketBuilder; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsgMetadata; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.queue.TbCoreConsumerStats; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 09.10.18. - */ -@Slf4j -@Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") -public class RemoteRuleEngineTransportService { - - @Value("${transport.remote.rule_engine.topic}") - private String ruleEngineTopic; - @Value("${transport.remote.notifications.topic}") - private String notificationsTopic; - @Value("${transport.remote.rule_engine.poll_interval}") - private int pollDuration; - @Value("${transport.remote.rule_engine.auto_commit_interval}") - private int autoCommitInterval; - - @Value("${transport.remote.rule_engine.poll_records_pack_size}") - private int pollRecordsPackSize; - @Value("${transport.remote.rule_engine.max_poll_records_per_second}") - private long pollRecordsPerSecond; - @Value("${transport.remote.rule_engine.max_poll_records_per_minute}") - private long pollRecordsPerMinute; - @Value("${transport.remote.rule_engine.stats.enabled:false}") - private boolean statsEnabled; - - @Autowired - private ActorSystemContext actorContext; - - //TODO: completely replace this routing with the Kafka routing by partition ids. - @Autowired - private ClusterRoutingService routingService; - @Autowired - private ClusterRpcService rpcService; - @Autowired - private DataDecodingEncodingService encodingService; - - @Autowired - private TbCoreQueueProvider coreQueueProvider; - - private TbQueueConsumer> ruleEngineConsumer; - - private TbQueueProducer> notificationsProducer; - - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-main-consumer")); - - private volatile boolean stopped = false; - - private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); - - @PostConstruct - public void init() { - notificationsProducer = coreQueueProvider.getTransportMsgProducer(); - notificationsProducer.init(); - - //TODO: 2.5 -// ruleEngineConsumer = -// ruleEngineConsumer.subscribe(); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting polling for events."); - LocalBucketBuilder builder = Bucket4j.builder(); - builder.addLimit(Bandwidth.simple(pollRecordsPerSecond, Duration.ofSeconds(1))); - builder.addLimit(Bandwidth.simple(pollRecordsPerMinute, Duration.ofMinutes(1))); - LocalBucket pollRateBucket = builder.build(); - BlockingBucket blockingPollRateBucket = pollRateBucket.asScheduler(); - - mainConsumerExecutor.execute(() -> { - while (!stopped) { - try { - List> msgs = ruleEngineConsumer.poll(pollDuration); - int recordsCount = msgs.size(); - if (recordsCount > 0) { - while (!blockingPollRateBucket.tryConsume(recordsCount, TimeUnit.SECONDS.toNanos(5))) { - log.info("Rule Engine consumer is busy. Required tokens: [{}]. Available tokens: [{}].", recordsCount, pollRateBucket.getAvailableTokens()); - Thread.sleep(TimeUnit.SECONDS.toMillis(1)); - } - log.trace("Processing {} records", recordsCount); - } - msgs.forEach(msg -> { - try { - ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); - if (toRuleEngineMsg.hasToDeviceActorMsg()) { - forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg()); - } - } catch (Throwable e) { - log.warn("Failed to process the notification.", e); - } - }); - } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - @Scheduled(fixedDelayString = "${transport.remote.rule_engine.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - stats.printStats(); - } - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg) { - process(nodeId, msg, null, null); - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - String topic = notificationsTopic + "." + nodeId; - UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); - ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); - log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); - //TODO: 2.5 id - TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(sessionId, transportMsg); - notificationsProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - log.trace("[{}] Pushing message to remote server: {}", address.get(), toDeviceActorMsg); - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - } else { - log.trace("Pushing message to local server: {}", toDeviceActorMsg); - actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); - } - } - - @PreDestroy - public void destroy() { - stopped = true; - if (ruleEngineConsumer != null) { - ruleEngineConsumer.unsubscribe(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - } - - private static class QueueCallbackAdaptor implements TbQueueCallback { - private final Runnable onSuccess; - private final Consumer onFailure; - - QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { - this.onSuccess = onSuccess; - this.onFailure = onFailure; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (onSuccess != null) { - onSuccess.run(); - } - } - - @Override - public void onFailure(Throwable t) { - if (onFailure != null) { - onFailure.accept(t); - } - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index 4765737430..e84e1b595e 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -44,7 +44,8 @@ import java.util.concurrent.*; */ @Slf4j @Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") +//TODO 2.5: This Confitional annotation should be removed, and Service renamed to something meaningful +//@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") public class RemoteTransportApiService { private final TbCoreQueueProvider tbCoreQueueProvider; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java deleted file mode 100644 index ac1c5965f3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; - -import java.io.IOException; - -/** - * Created by ashvayka on 05.10.18. - */ -public class TransportApiRequestDecoder implements TbKafkaDecoder { - @Override - public TransportApiRequestMsg decode(byte[] data) throws IOException { - return TransportApiRequestMsg.parseFrom(data); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java deleted file mode 100644 index 35f8a7b1aa..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.kafka.TbKafkaEncoder; - -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; - -/** - * Created by ashvayka on 05.10.18. - */ -public class TransportApiResponseEncoder implements TbKafkaEncoder { - @Override - public byte[] encode(TransportApiResponseMsg value) { - return value.toByteArray(); - } -} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index da2e5beb98..57d837de31 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -528,7 +528,7 @@ swagger: version: "${SWAGGER_VERSION:2.0}" queue: - type: "${TB_QUEUE_TYPE:kafka}" + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -556,6 +556,7 @@ queue: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:100}" + pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java index d717169048..0c34bbf9d6 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,83 +21,92 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.util.ReflectionTestUtils; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; +import org.thingsboard.server.discovery.ConsistentHashPartitionService; +import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.discovery.TbServiceInfoProvider; +import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.stream.Collectors; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) public class ConsistentClusterRoutingServiceTest { - private ConsistentClusterRoutingService clusterRoutingService; + public static final int ITERATIONS = 1000000; + private ConsistentHashPartitionService clusterRoutingService; - private DiscoveryService discoveryService; + private TbServiceInfoProvider discoveryService; private String hashFunctionName = "murmur3_128"; - private Integer virtualNodesSize = 1024*4; - private ServerAddress currentServer = new ServerAddress(" 100.96.1.0", 9001, ServerType.CORE); + private Integer virtualNodesSize = 16; @Before public void setup() throws Exception { - discoveryService = mock(DiscoveryService.class); - clusterRoutingService = new ConsistentClusterRoutingService(); - ReflectionTestUtils.setField(clusterRoutingService, "discoveryService", discoveryService); + discoveryService = mock(TbServiceInfoProvider.class); + clusterRoutingService = new ConsistentHashPartitionService(discoveryService); + ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); + ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); + ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); + ReflectionTestUtils.setField(clusterRoutingService, "ruleEnginePartitions", 100); + ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); - when(discoveryService.getCurrentServer()).thenReturn(new ServerInstance(currentServer)); - List otherServers = new ArrayList<>(); + TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() + .setServiceId("100.96.1.1") + .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) + .build(); +// when(discoveryService.getServiceInfo()).thenReturn(currentServer); + List otherServers = new ArrayList<>(); for (int i = 1; i < 30; i++) { - otherServers.add(new ServerInstance(new ServerAddress(" 100.96." + i*2 + "." + i, 9001, ServerType.CORE))); + otherServers.add(TransportProtos.ServiceInfo.newBuilder() + .setServiceId("100.96." + i * 2 + "." + i) + .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) + .build()); } - when(discoveryService.getOtherServers()).thenReturn(otherServers); clusterRoutingService.init(); + clusterRoutingService.recalculatePartitions(currentServer, otherServers); } @Test public void testDispersionOnMillionDevices() { List devices = new ArrayList<>(); - for (int i = 0; i < 1000000; i++) { + for (int i = 0; i < ITERATIONS; i++) { devices.add(new DeviceId(UUIDs.timeBased())); } - testDevicesDispersion(devices); } private void testDevicesDispersion(List devices) { long start = System.currentTimeMillis(); - Map map = new HashMap<>(); + Map map = new HashMap<>(); for (DeviceId deviceId : devices) { - ServerAddress address = clusterRoutingService.resolveById(deviceId).orElse(currentServer); - map.put(address, map.getOrDefault(address, 0) + 1); + TopicPartitionInfo address = clusterRoutingService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, deviceId); + Integer partition = address.getPartition().get(); + map.put(partition, map.getOrDefault(partition, 0) + 1); } - List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); + List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); long end = System.currentTimeMillis(); - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + (data.get(data.size() - 1).getValue() - data.get(0).getValue())); + double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); + System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff/ITERATIONS) * 100.0) + "%)"); - for (Map.Entry entry : data) { -// System.out.println(entry.getKey().getHost() + ": " + entry.getValue()); + for (Map.Entry entry : data) { + System.out.println(entry.getKey() + ": " + entry.getValue()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java index 26d90a26c6..b3778c2f8e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ */ package org.thingsboard.server; -import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.discovery.TopicPartitionInfo; public interface TbQueueProducer { @@ -23,10 +23,6 @@ public interface TbQueueProducer { String getDefaultTopic(); - void send(T msg, TbQueueCallback callback); - - void send(String topic, T msg, TbQueueCallback callback); - - ListenableFuture send(String topic, int partition, T msg, TbQueueCallback callback); + void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback); } diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java index d022dafadc..53914953f3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -28,6 +28,7 @@ import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.discovery.TopicPartitionInfo; import java.util.List; import java.util.UUID; @@ -160,7 +161,7 @@ public class DefaultTbQueueRequestTemplate responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime); - requestTemplate.send(request, new TbQueueCallback() { + requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { log.trace("[{}] Request sent: {}", requestId, metadata); diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java index 3804fb5160..c77692ee36 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java @@ -23,6 +23,7 @@ import org.thingsboard.server.TbQueueHandler; import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueResponseTemplate; +import org.thingsboard.server.discovery.TopicPartitionInfo; import java.util.List; import java.util.UUID; @@ -108,7 +109,7 @@ public class DefaultTbQueueResponseTemplate { pendingRequestCount.decrementAndGet(); response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); - responseTemplate.send(responseTopic, response, null); + responseTemplate.send(TopicPartitionInfo.builder().topic(responseTopic).build(), response, null); }, e -> { pendingRequestCount.decrementAndGet(); diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java similarity index 66% rename from application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java rename to common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java index 711677dc78..f96fbd4ece 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.routing; +package org.thingsboard.server.discovery; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -25,11 +24,10 @@ import java.util.concurrent.ConcurrentSkipListMap; * Created by ashvayka on 23.09.18. */ @Slf4j -public class ConsistentHashCircle { - private final ConcurrentNavigableMap circle = - new ConcurrentSkipListMap<>(); +public class ConsistentHashCircle { + private final ConcurrentNavigableMap circle = new ConcurrentSkipListMap<>(); - public void put(long hash, ServerInstance instance) { + public void put(long hash, T instance) { circle.put(hash, instance); } @@ -45,7 +43,7 @@ public class ConsistentHashCircle { return circle.containsKey(hash); } - public ConcurrentNavigableMap tailMap(Long hash) { + public ConcurrentNavigableMap tailMap(Long hash) { return circle.tailMap(hash); } @@ -53,11 +51,11 @@ public class ConsistentHashCircle { return circle.firstKey(); } - public ServerInstance get(Long hash) { + public T get(Long hash) { return circle.get(hash); } public void log() { - circle.entrySet().forEach((e) -> log.debug("{} -> {}", e.getKey(), e.getValue().getServerAddress())); + circle.forEach((key, value) -> log.debug("{} -> {}", key, value)); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java new file mode 100644 index 0000000000..c9b7b04e70 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java @@ -0,0 +1,221 @@ +package org.thingsboard.server.discovery; + +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; + +import javax.annotation.PostConstruct; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentNavigableMap; + +@Service +@Slf4j +public class ConsistentHashPartitionService implements PartitionService { + + @Value("${queue.core.topic}") + private String coreTopic; + @Value("${queue.core.partitions:100}") + private Integer corePartitions; + @Value("${queue.rule_engine.topic}") + private String ruleEngineTopic; + @Value("${queue.rule_engine.partitions:100}") + private Integer ruleEnginePartitions; + @Value("${queue.partitions.hash_function_name:murmur3_32}") + private String hashFunctionName; + @Value("${queue.partitions.virtual_nodes_size:16}") + private Integer virtualNodesSize; + + private final TbServiceInfoProvider serviceInfoProvider; + private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); + private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + //TODO: Fetch this from the database, together with size of partitions for each service for each tenant. + private ConcurrentMap> isolatedTenants = new ConcurrentHashMap<>(); + + private HashFunction hashFunction; + + public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider) { + this.serviceInfoProvider = serviceInfoProvider; + } + + @PostConstruct + public void init() { + this.hashFunction = forName(hashFunctionName); + partitionSizes.put(ServiceType.TB_CORE, corePartitions); + partitionSizes.put(ServiceType.TB_RULE_ENGINE, ruleEnginePartitions); + partitionTopics.put(ServiceType.TB_CORE, coreTopic); + partitionTopics.put(ServiceType.TB_RULE_ENGINE, ruleEngineTopic); + } + + @Override + public List getCurrentPartitions(ServiceType serviceType) { + ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); + TenantId tenantId = getTenantId(currentService); + ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); + List partitions = myPartitions.get(serviceKey); + List topicPartitions = new ArrayList<>(); + for (Integer partition : partitions) { + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); + tpi.topic(partitionTopics.get(serviceType)); + tpi.partition(partition); + if (!tenantId.isNullUid()) { + tpi.tenantId(tenantId); + } + topicPartitions.add(tpi.build()); + } + return topicPartitions; + } + + @Override + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); + int hash = hashFunction.newHasher() + .putLong(entityId.getId().getMostSignificantBits()) + .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); + int partition = Math.abs(hash % partitionSizes.get(serviceType)); + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); + tpi.topic(partitionTopics.get(serviceType)); + tpi.partition(partition); + if (isolated) { + tpi.tenantId(tenantId); + } + return tpi.build(); + } + + @Override + public void recalculatePartitions(ServiceInfo currentService, List otherServices) { + logServiceInfo(currentService); + otherServices.forEach(this::logServiceInfo); + + Map> newCircles = new HashMap<>(ServiceType.values().length); + for (ServiceType serverType : ServiceType.values()) { + newCircles.put(serverType, new ConsistentHashCircle<>()); + } + addNode(newCircles, currentService); + for (ServiceInfo other : otherServices) { + addNode(newCircles, other); + TenantId tenantId = getTenantId(other); + if (!tenantId.isNullUid()) { + isolatedTenants.putIfAbsent(tenantId, new HashSet<>()); + for (String serviceType : other.getServiceTypesList()) { + isolatedTenants.get(tenantId).add(ServiceType.valueOf(serviceType.toUpperCase())); + } + + } + } + ConcurrentMap> oldPartitions = myPartitions; + myPartitions = new ConcurrentHashMap<>(); + partitionSizes.forEach((type, size) -> { + for (int i = 0; i < size; i++) { + ServiceInfo serviceInfo = resolveByPartitionIdx(newCircles.get(type), i); + if (currentService.equals(serviceInfo)) { + myPartitions.putIfAbsent(new ServiceKey(type, getTenantId(serviceInfo)), new ArrayList<>()); + } + } + }); + myPartitions.forEach((serviceKey, partitions) -> { + if (!partitions.equals(oldPartitions.get(serviceKey))) { + log.info("[{}] NEW PARTITIONS: {}", serviceKey, partitions); + } + }); + } + + private void logServiceInfo(TransportProtos.ServiceInfo server) { + TenantId tenantId = getTenantId(server); + if (tenantId.isNullUid()) { + log.info("[{}] Found common server: [{}]", server.getServiceId(), server.getServiceTypesList()); + } else { + log.info("[{}][{}] Found specific server: [{}]", server.getServiceId(), tenantId, server.getServiceTypesList()); + } + } + + private TenantId getTenantId(TransportProtos.ServiceInfo serviceInfo) { + return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB())); + } + + private void addNode(Map> circles, ServiceInfo instance) { + for (String serviceTypeStr : instance.getServiceTypesList()) { + ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + for (int i = 0; i < virtualNodesSize; i++) { + circles.get(serviceType).put(hash(instance, i).asLong(), instance); + } + } + } + + private ServiceInfo resolveByPartitionIdx(ConsistentHashCircle circle, Integer partitionIdx) { + if (circle.isEmpty()) { + return null; + } + Long hash = hashFunction.newHasher().putInt(partitionIdx).hash().asLong(); + if (!circle.containsKey(hash)) { + ConcurrentNavigableMap tailMap = circle.tailMap(hash); + hash = tailMap.isEmpty() ? + circle.firstKey() : tailMap.firstKey(); + } + return circle.get(hash); + } + + private HashCode hash(ServiceInfo instance, int i) { + return hashFunction.newHasher().putString(instance.getServiceId(), StandardCharsets.UTF_8).putInt(i).hash(); + } + + private static class ServiceKey { + @Getter + private final ServiceType serviceType; + @Getter + private final TenantId tenantId; + + public ServiceKey(ServiceType serviceType, TenantId tenantId) { + this.serviceType = serviceType; + this.tenantId = tenantId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceKey that = (ServiceKey) o; + return serviceType == that.serviceType && + Objects.equals(tenantId, that.tenantId); + } + + @Override + public int hashCode() { + return Objects.hash(serviceType, tenantId); + } + } + + public static HashFunction forName(String name) { + switch (name) { + case "murmur3_32": + return Hashing.murmur3_32(); + case "murmur3_128": + return Hashing.murmur3_128(); + case "crc32": + return Hashing.crc32(); + case "md5": + return Hashing.md5(); + default: + throw new IllegalArgumentException("Can't find hash function with name " + name); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java index 0b09a333d8..1fd87a40b6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -69,11 +69,14 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { ServiceInfo.Builder builder = ServiceInfo.newBuilder() .setServiceId(serviceId) .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())); + UUID tenantId; if (!StringUtils.isEmpty(tenantIdStr)) { - UUID tenantId = UUID.fromString(tenantIdStr); - builder.setTenantIdMSB(tenantId.getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); + tenantId = UUID.fromString(tenantIdStr); + } else { + tenantId = TenantId.NULL_UUID; } + builder.setTenantIdMSB(tenantId.getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java new file mode 100644 index 0000000000..ef9aca72a0 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java @@ -0,0 +1,32 @@ +package org.thingsboard.server.discovery; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.annotation.DependsOn; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; + +import java.util.Collections; + +@Service +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +@Slf4j +@DependsOn("environmentLogService") +public class DummyDiscoveryService implements PartitionDiscoveryService { + + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; + + + public DummyDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.partitionService = partitionService; + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList()); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java index d85b1d3f44..6fe4571517 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java @@ -15,15 +15,6 @@ */ package org.thingsboard.server.discovery; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.List; - public interface PartitionDiscoveryService { - List getCurrentPartitions(ServiceType serviceType); - - TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); - } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java new file mode 100644 index 0000000000..90d43f4af3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java @@ -0,0 +1,16 @@ +package org.thingsboard.server.discovery; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.List; + +public interface PartitionService { + + List getCurrentPartitions(ServiceType serviceType); + + TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + + void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java index b46b941360..4480d07dd6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java @@ -15,12 +15,27 @@ */ package org.thingsboard.server.discovery; -import lombok.Data; +import lombok.Builder; +import org.thingsboard.server.common.data.id.TenantId; -@Data +import java.util.Optional; + +@Builder public class TopicPartitionInfo { - private String topic; - private int partition; + private final String topic; + private final TenantId tenantId; + private final Integer partition; + + public String getTopic() { + return topic; + } + + public Optional getTenantId() { + return Optional.ofNullable(tenantId); + } + public Optional getPartition() { + return Optional.ofNullable(partition); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java index a722f2f395..d9af5bb894 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,11 +15,13 @@ */ package org.thingsboard.server.discovery; +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.SerializationUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; @@ -39,16 +41,21 @@ import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.transport.TransportProtos; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.Collections; import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED; @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) @@ -66,16 +73,8 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P @Value("${zk.zk_dir}") private String zkDir; - @Value("${queue.core.partitions:100}") - private Integer corePartitions; - @Value("${queue.rule_engine.partitions:100}") - private Integer ruleEnginePartitions; - - @Autowired - private TbServiceInfoProvider serviceIdProvider; - - private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); - private final ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; private ExecutorService reconnectExecutorService; private CuratorFramework client; @@ -85,14 +84,9 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P private volatile boolean stopped = true; - @Override - public List getCurrentPartitions(ServiceType serviceType) { - return Collections.emptyList(); - } - - @Override - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { - + public ZkPartitionDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.partitionService = partitionService; } @PostConstruct @@ -105,15 +99,26 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); - partitionSizes.put(ServiceType.TB_CORE, corePartitions); - partitionSizes.put(ServiceType.TB_RULE_ENGINE, ruleEnginePartitions); - log.info("Initializing discovery service using ZK connect string: {}", zkUrl); zkNodesDir = zkDir + "/nodes"; initZkClient(); } + private List getOtherServers() { + return cache.getCurrentData().stream() + .filter(cd -> !cd.getPath().equals(nodePath)) + .map(cd -> { + try { + return TransportProtos.ServiceInfo.parseFrom(cd.getData()); + } catch (NoSuchElementException | InvalidProtocolBufferException e) { + log.error("Failed to decode ZK node", e); + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } + @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent event) { if (stopped) { @@ -127,23 +132,20 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P return; } publishCurrentServer(); - getOtherServers().forEach( - server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) - ); + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); } - @Override public synchronized void publishCurrentServer() { - ServerInstance self = this.serverInstance.getSelf(); + TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); if (currentServerExists()) { - log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); + log.info("[{}] ZK node for current instance already exists, NOT created new one: {}", self.getServiceId(), nodePath); } else { try { - log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); + log.info("[{}] Creating ZK node for current instance", self.getServiceId()); nodePath = client.create() .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); - log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); + .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", self.toByteArray()); + log.info("[{}] Created ZK node for current instance: {}", self.getServiceId(), nodePath); client.getConnectionStateListenable().addListener(checkReconnect(self)); } catch (Exception e) { log.error("Failed to create ZK node", e); @@ -157,10 +159,10 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P return false; } try { - ServerInstance self = this.serverInstance.getSelf(); - ServerAddress registeredServerAdress = null; - registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); - if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { + TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); + TransportProtos.ServiceInfo registeredServerInfo = null; + registeredServerInfo = TransportProtos.ServiceInfo.parseFrom(client.getData().forPath(nodePath)); + if (self.equals(registeredServerInfo)) { return true; } } catch (KeeperException.NoNodeException e) { @@ -171,9 +173,9 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P return false; } - private ConnectionStateListener checkReconnect(ServerInstance self) { + private ConnectionStateListener checkReconnect(TransportProtos.ServiceInfo self) { return (client, newState) -> { - log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); + log.info("[{}] ZK state changed: {}", self.getServiceId(), newState); if (newState == ConnectionState.LOST) { reconnectExecutorService.submit(this::reconnect); } @@ -250,6 +252,46 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P @Override public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent pathChildrenCacheEvent) throws Exception { - + if (stopped) { + log.debug("Ignoring {}. Service is stopped.", pathChildrenCacheEvent); + return; + } + if (client.getState() != CuratorFrameworkState.STARTED) { + log.debug("Ignoring {}, ZK client is not started, ZK client state [{}]", pathChildrenCacheEvent, client.getState()); + return; + } + ChildData data = pathChildrenCacheEvent.getData(); + if (data == null) { + log.debug("Ignoring {} due to empty child data", pathChildrenCacheEvent); + return; + } else if (data.getData() == null) { + log.debug("Ignoring {} due to empty child's data", pathChildrenCacheEvent); + return; + } else if (nodePath != null && nodePath.equals(data.getPath())) { + if (pathChildrenCacheEvent.getType() == CHILD_REMOVED) { + log.info("ZK node for current instance is somehow deleted."); + publishCurrentServer(); + } + log.debug("Ignoring event about current server {}", pathChildrenCacheEvent); + return; + } + TransportProtos.ServiceInfo instance; + try { + instance = TransportProtos.ServiceInfo.parseFrom(data.getData()); + } catch (InvalidProtocolBufferException e) { + log.error("Failed to decode server instance for node {}", data.getPath(), e); + throw e; + } + log.info("Processing [{}] event for [{}]", pathChildrenCacheEvent.getType(), instance.getServiceId()); + switch (pathChildrenCacheEvent.getType()) { + case CHILD_ADDED: + case CHILD_UPDATED: + case CHILD_REMOVED: + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); + break; + default: + break; + } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java b/common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java rename to common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java index a6a881ecde..cff3fd1056 100644 --- a/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java +++ b/common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.environment; +package org.thingsboard.server.environment; import lombok.extern.slf4j.Slf4j; import org.apache.zookeeper.Environment; @@ -33,7 +33,7 @@ public class EnvironmentLogService { @PostConstruct public void init() { - Environment.logEnv("Thingsboard server environment: ", log); + Environment.logEnv("ThingsBoard server environment: ", log); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java index d76a2dc625..8072a2da66 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -29,6 +29,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.discovery.TopicPartitionInfo; import java.util.List; import java.util.Properties; @@ -46,10 +47,6 @@ public class TBKafkaProducerTemplate implements TbQueuePro private final KafkaProducer producer; - private final TbKafkaPartitioner partitioner; - - private ConcurrentMap> partitionInfoMap; - @Getter private final String defaultTopic; @@ -66,41 +63,28 @@ public class TBKafkaProducerTemplate implements TbQueuePro } this.settings = settings; this.producer = new KafkaProducer<>(props); - this.partitioner = partitioner; this.defaultTopic = defaultTopic; } - public void init() { - this.partitionInfoMap = new ConcurrentHashMap<>(); - if (!StringUtils.isEmpty(defaultTopic)) { - try { - TBKafkaAdmin admin = new TBKafkaAdmin(this.settings); - admin.waitForTopic(defaultTopic, 30, TimeUnit.SECONDS); - log.info("[{}] Topic exists.", defaultTopic); - } catch (Exception e) { - log.info("[{}] Failed to wait for topic: {}", defaultTopic, e.getMessage(), e); - throw new RuntimeException(e); - } - //Maybe this should not be cached, but we don't plan to change size of partitions - this.partitionInfoMap.putIfAbsent(defaultTopic, producer.partitionsFor(defaultTopic)); - } - } - @Override - public void send(T msg, TbQueueCallback callback) { - send(defaultTopic, msg, callback); + public void init() { } @Override - public void send(String topic, T msg, TbQueueCallback callback) { + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { String key = msg.getKey().toString(); byte[] data = msg.getData(); ProducerRecord record; Iterable
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); - - Integer partition = getPartition(topic, msg); - record = new ProducerRecord<>(topic, partition, key, data, headers); - Future result = producer.send(record, (metadata, exception) -> { + StringBuilder topic = new StringBuilder().append(tpi.getTopic()); + if (tpi.getTenantId().isPresent()) { + topic.append(".").append(tpi.getTenantId().get().getId().toString()); + } + if (tpi.getPartition().isPresent()) { + topic.append(".").append(tpi.getPartition().get()); + } + record = new ProducerRecord<>(topic.toString(), null, key, data, headers); + producer.send(record, (metadata, exception) -> { if (exception == null) { callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); } else { @@ -108,12 +92,4 @@ public class TBKafkaProducerTemplate implements TbQueuePro } }); } - - private Integer getPartition(String topic, T value) { - if (partitioner == null) { - return null; - } else { - return partitioner.partition(topic, value.getKey().toString(), value, value.getData(), partitionInfoMap.computeIfAbsent(topic, producer::partitionsFor)); - } - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java index e71d579c7f..8abb5e2d43 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,6 +19,7 @@ import lombok.Data; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.discovery.TopicPartitionInfo; @Data public class InMemoryTbQueueProducer implements TbQueueProducer { @@ -37,18 +38,8 @@ public class InMemoryTbQueueProducer implements TbQueuePro } @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(T msg, TbQueueCallback callback) { - send(defaultTopic, msg, callback); - } - - @Override - public void send(String topic, T msg, TbQueueCallback callback) { - boolean result = storage.put(topic, msg); + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + boolean result = storage.put(tpi.getTopic(), msg); if (result) { callback.onSuccess(null); } else { diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java index 79f5dd7da0..c7839e70e9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java @@ -32,12 +32,12 @@ import org.thingsboard.server.memory.InMemoryTbQueueProducer; @Slf4j @Component -@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && '${queue.type:null}'=='in-memory'") -public class InMemoryTbCoreQueueProvider implements TbCoreQueueProvider { +@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") +public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { private final TbQueueCoreSettings coreSettings; - public InMemoryTbCoreQueueProvider(TbQueueCoreSettings coreSettings) { + public InMemoryMonolithQueueProvider(TbQueueCoreSettings coreSettings) { this.coreSettings = coreSettings; } @@ -59,6 +59,12 @@ public class InMemoryTbCoreQueueProvider implements TbCoreQueueProvider { return producer; } + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); + return consumer; + } + @Override public TbQueueConsumer> getToCoreMsgConsumer() { InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java new file mode 100644 index 0000000000..6aaf76420b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") +public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { + + private final TbKafkaSettings kafkaSettings; + private final TbNodeIdProvider nodeIdProvider; + private final TbQueueCoreSettings coreSettings; + + public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + this.kafkaSettings = kafkaSettings; + this.nodeIdProvider = nodeIdProvider; + this.coreSettings = coreSettings; + } + + @Override + public TbQueueProducer> getTransportMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(coreSettings.getTopic()); + responseBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("tb-rule-engine-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return responseBuilder.build(); + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getTopic()); + consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("tb-core-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(coreSettings.getTopic()); + responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + //TODO 2.5 + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + return responseBuilder.build(); + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-api-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index d830cff8fa..426a30eea1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -32,7 +32,7 @@ import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; @Component -@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && '${queue.type:null}'=='kafka'") +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final TbKafkaSettings kafkaSettings; @@ -49,7 +49,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { public TbQueueProducer> getTransportMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -58,7 +58,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -74,28 +74,33 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { @Override public TbQueueConsumer> getToCoreMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getTopic()); + consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("tb-core-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(coreSettings.getTopic()); responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); responseBuilder.autoCommit(true); - //TODO: 2.5 -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); -// responseBuilder.decoder(new TransportApiResponseDecoder()); + //TODO 2.5 + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); return responseBuilder.build(); } - @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { - return null; - } - @Override public TbQueueProducer> getTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("transport-api-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java new file mode 100644 index 0000000000..6057ebba6f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") +public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { + + private final TbKafkaSettings kafkaSettings; + private final TbNodeIdProvider nodeIdProvider; + private final TbQueueCoreSettings coreSettings; + + public KafkaTbRuleEngineQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + this.kafkaSettings = kafkaSettings; + this.nodeIdProvider = nodeIdProvider; + this.coreSettings = coreSettings; + } + + @Override + public TbQueueProducer> getTransportMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(coreSettings.getTopic()); + responseBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("tb-rule-engine-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return responseBuilder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index 3b671a3adb..c7b5b6f839 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -35,7 +35,7 @@ import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; @Component -@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && '${queue.type:null}'=='kafka'") +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j public class KafkaTransportQueueProvider implements TransportQueueProvider { @@ -62,9 +62,7 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); responseBuilder.autoCommit(true); - //TODO: 2.5 -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); -// responseBuilder.decoder(new TransportApiResponseDecoder()); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java new file mode 100644 index 0000000000..46f1a3351d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Core Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbRuleEngineQueueProvider { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> getTransportMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreMsgProducer(); + + /** + * Used to consume messages by TB Core Service + * + * @return + */ + TbQueueConsumer> getToRuleEngineMsgConsumer(); + +} diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index d6dcbbabc6..af471e537c 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -36,6 +36,10 @@ + + org.thingsboard.common + queue + org.thingsboard.common data diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index cdf6b3f763..661411cf89 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -34,6 +34,9 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.discovery.PartitionService; +import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.discovery.TopicPartitionInfo; import org.thingsboard.server.provider.TransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -76,8 +79,8 @@ public class DefaultTransportService implements TransportService { @Value("${transport.sessions.report_timeout}") private long sessionReportTimeout; - @Autowired - private TransportQueueProvider queueProvider; + private final TransportQueueProvider queueProvider; + private final PartitionService partitionService; @Value("${kafka.notifications.poll_interval}") private int notificationsPollDuration; @@ -98,6 +101,11 @@ public class DefaultTransportService implements TransportService { private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); private volatile boolean stopped = false; + public DefaultTransportService(TransportQueueProvider queueProvider, PartitionService partitionService) { + this.queueProvider = queueProvider; + this.partitionService = partitionService; + } + @PostConstruct public void init() { if (rateLimitEnabled) { @@ -420,6 +428,14 @@ public class DefaultTransportService implements TransportService { return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()); } + protected TenantId getTenantId(TransportProtos.SessionInfoProto sessionInfo) { + return new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + } + + protected DeviceId getDeviceId(TransportProtos.SessionInfoProto sessionInfo) { + return new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + } + public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { return TransportProtos.SessionEventMsg.newBuilder() .setSessionType(TransportProtos.SessionType.ASYNC) @@ -427,15 +443,19 @@ public class DefaultTransportService implements TransportService { } protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { - tbCoreMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), - ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? - new TransportTbQueueCallback(callback) : null); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); + tbCoreMsgProducer.send(tpi, + new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? + new TransportTbQueueCallback(callback) : null); } protected void sendToRuleEngine(TransportProtos.SessionInfoProto sessionInfo, TransportToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), - ToRuleEngineMsg.newBuilder().setToRuleEngineMsg(toRuleEngineMsg).build()), callback != null ? - new TransportTbQueueCallback(callback) : null); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); + ruleEngineMsgProducer.send(tpi, + new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToRuleEngineMsg.newBuilder().setToRuleEngineMsg(toRuleEngineMsg).build()), callback != null ? + new TransportTbQueueCallback(callback) : null); } private class TransportTbQueueCallback implements TbQueueCallback {