diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 54ac41172c..7e9b68882b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -32,12 +32,10 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.common.data.DataConstants; @@ -45,10 +43,11 @@ import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -65,24 +64,23 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.kafka.TbNodeIdProvider; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; import org.thingsboard.server.service.mail.MailExecutorService; -import org.thingsboard.server.service.rpc.DeviceRpcService; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; +import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.script.JsExecutorService; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.session.DeviceSessionCacheService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.service.transport.RuleEngineTransportService; +import org.thingsboard.server.service.transport.TbCoreToTransportService; import javax.annotation.Nullable; import java.io.IOException; @@ -106,35 +104,24 @@ public class ActorSystemContext { return debugPerTenantLimits; } + @Autowired @Getter @Setter - private ActorService actorService; + private TbServiceInfoProvider serviceInfoProvider; - @Autowired @Getter - private DiscoveryService discoveryService; + @Setter + private ActorService actorService; @Autowired @Getter @Setter private ComponentDiscoveryService componentService; - @Autowired - @Getter - private ClusterRoutingService routingService; - - @Autowired - @Getter - private ClusterRpcService rpcService; - @Autowired @Getter private DataDecodingEncodingService encodingService; - @Autowired - @Getter - private DeviceAuthService deviceAuthService; - @Autowired @Getter private DeviceService deviceService; @@ -163,6 +150,13 @@ public class ActorSystemContext { @Getter private RuleChainService ruleChainService; + @Autowired + private PartitionService partitionService; + + @Autowired + @Getter + private TbClusterService clusterService; + @Autowired @Getter private TimeseriesService tsService; @@ -195,10 +189,6 @@ public class ActorSystemContext { @Getter private TelemetrySubscriptionService tsSubService; - @Autowired - @Getter - private DeviceRpcService deviceRpcService; - @Autowired @Getter private JsInvokeService jsSandbox; @@ -211,10 +201,6 @@ public class ActorSystemContext { @Getter private MailExecutorService mailExecutor; - @Autowired - @Getter - private ClusterRpcCallbackExecutorService clusterRpcCallbackExecutor; - @Autowired @Getter private DbCallbackExecutorService dbCallbackExecutor; @@ -231,27 +217,32 @@ public class ActorSystemContext { @Getter private MailService mailService; - @Autowired + //TODO: separate context for TbCore and TbRuleEngine + @Autowired(required = false) @Getter private DeviceStateService deviceStateService; - @Autowired + @Autowired(required = false) @Getter private DeviceSessionCacheService deviceSessionCacheService; - @Lazy - @Autowired + @Autowired(required = false) @Getter - private RuleEngineTransportService ruleEngineTransportService; + private TbCoreToTransportService tbCoreToTransportService; - @Lazy - @Autowired + /** + * The following Service will be null if we operate in tb-core mode + */ + @Autowired(required = false) @Getter - private RuleChainTransactionService ruleChainTransactionService; + private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; - @Value("${cluster.partition_id}") + /** + * The following Service will be null if we operate in tb-rule-engine mode + */ + @Autowired(required = false) @Getter - private long queuePartitionId; + private TbCoreDeviceRpcService tbCoreDeviceRpcService; @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter @@ -269,10 +260,6 @@ public class ActorSystemContext { @Getter private long queuePersistenceTimeout; - @Value("${actors.client_side_rpc.timeout}") - @Getter - private long clientSideRpcTimeout; - @Value("${actors.rule.chain.error_persist_frequency}") @Getter private long ruleChainErrorPersistFrequency; @@ -334,11 +321,6 @@ public class ActorSystemContext { @Setter private ActorSystem actorSystem; - @Autowired - @Getter - private TbNodeIdProvider nodeIdProvider; - - @Getter @Setter private ActorRef appActor; @@ -365,6 +347,8 @@ public class ActorSystemContext { config = ConfigFactory.parseResources(AKKA_CONF_FILE_NAME).withFallback(ConfigFactory.load()); } + + public Scheduler getScheduler() { return actorSystem.scheduler(); } @@ -374,7 +358,7 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.ERROR); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); + event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e))); persistEvent(event); } @@ -383,7 +367,7 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); + event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -397,8 +381,8 @@ public class ActorSystemContext { return sw.toString(); } - private JsonNode toBodyJson(ServerAddress server, ComponentLifecycleEvent event, Optional e) { - ObjectNode node = mapper.createObjectNode().put("server", server.toString()).put("event", event.name()); + private JsonNode toBodyJson(String serviceId, ComponentLifecycleEvent event, Optional e) { + ObjectNode node = mapper.createObjectNode().put("server", serviceId).put("event", event.name()); if (e.isPresent()) { node = node.put("success", false); node = node.put("error", toString(e.get())); @@ -408,12 +392,21 @@ public class ActorSystemContext { return node; } - private JsonNode toBodyJson(ServerAddress server, String method, String body) { - return mapper.createObjectNode().put("server", server.toString()).put("method", method).put("error", body); + private JsonNode toBodyJson(String serviceId, String method, String body) { + return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); } - public String getServerAddress() { - return discoveryService.getCurrentServer().getServerAddress().toString(); + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + return partitionService.resolve(serviceType, tenantId, entityId); + } + + public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { + return partitionService.resolve(serviceType, queueName, tenantId, entityId); + } + + + public String getServiceId() { + return serviceInfoProvider.getServiceId(); } public void persistDebugInput(TenantId tenantId, EntityId entityId, TbMsg tbMsg, String relationType) { @@ -444,7 +437,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() .put("type", type) - .put("server", getServerAddress()) + .put("server", getServiceId()) .put("entityId", tbMsg.getOriginator().getId().toString()) .put("entityName", tbMsg.getOriginator().getEntityType().name()) .put("msgId", tbMsg.getId().toString()) @@ -504,7 +497,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() //todo: what fields are needed here? - .put("server", getServerAddress()) + .put("server", getServiceId()) .put("message", "Reached debug mode rate limit!"); if (error != null) { @@ -530,4 +523,7 @@ public class ActorSystemContext { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } + public void tell(TbActorMsg tbActorMsg, ActorRef sender) { + appActor.tell(tbActorMsg, sender); + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 778c945a32..b7782a6a87 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -20,19 +20,13 @@ import akka.actor.LocalActorRef; import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; -import akka.actor.SupervisorStrategy.Directive; import akka.actor.Terminated; -import akka.event.Logging; -import akka.event.LoggingAdapter; -import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; +import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.actors.shared.rulechain.SystemRuleChainManager; import org.thingsboard.server.actors.tenant.TenantActor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; @@ -42,29 +36,32 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import scala.concurrent.duration.Duration; -import java.util.HashMap; -import java.util.Map; +import java.util.HashSet; import java.util.Optional; +import java.util.Set; -public class AppActor extends RuleChainManagerActor { +public class AppActor extends ContextAwareActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); private final TenantService tenantService; private final BiMap tenantActors; + private final Set deletedTenants; private boolean ruleChainsInitialized; private AppActor(ActorSystemContext systemContext) { - super(systemContext, new SystemRuleChainManager(systemContext)); + super(systemContext); this.tenantService = systemContext.getTenantService(); this.tenantActors = HashBiMap.create(); + this.deletedTenants = new HashSet<>(); } @Override @@ -79,7 +76,7 @@ public class AppActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { if (!ruleChainsInitialized) { - initRuleChainsAndTenantActors(); + initTenantActors(); ruleChainsInitialized = true; if (msg.getMsgType() != MsgType.APP_INIT_MSG) { log.warn("Rule Chains initialized by unexpected message: {}", msg); @@ -88,17 +85,14 @@ public class AppActor extends RuleChainManagerActor { switch (msg.getMsgType()) { case APP_INIT_MSG: break; - case SEND_TO_CLUSTER_MSG: - onPossibleClusterMsg((SendToClusterMsg) msg); - break; - case CLUSTER_EVENT_MSG: + case PARTITION_CHANGE_MSG: broadcast(msg); break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case SERVICE_TO_RULE_ENGINE_MSG: - onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); + case QUEUE_TO_RULE_ENGINE_MSG: + onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -106,7 +100,6 @@ public class AppActor extends RuleChainManagerActor { case DEVICE_NAME_OR_TYPE_UPDATE_TO_DEVICE_ACTOR_MSG: case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: - case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onToDeviceActorMsg((TenantAwareMsg) msg); break; default: @@ -115,16 +108,30 @@ public class AppActor extends RuleChainManagerActor { return true; } - private void initRuleChainsAndTenantActors() { + private void initTenantActors() { log.info("Starting main system actor."); try { - initRuleChains(); - if (systemContext.isTenantComponentsInitEnabled()) { - PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); - for (Tenant tenant : tenantIterator) { + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + if (isolatedTenantId.isPresent()) { + Tenant tenant = systemContext.getTenantService().findTenantById(isolatedTenantId.get()); + if (tenant != null) { log.debug("[{}] Creating tenant actor", tenant.getId()); getOrCreateTenantActor(tenant.getId()); log.debug("Tenant actor created."); + } else { + log.error("[{}] Tenant with such ID does not exist", isolatedTenantId.get()); + } + } else if (systemContext.isTenantComponentsInitEnabled()) { + PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); + boolean isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + boolean isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + for (Tenant tenant : tenantIterator) { + if (isCore || (isRuleEngine && !tenant.isIsolatedTbRuleEngine())) { + log.debug("[{}] Creating tenant actor", tenant.getId()); + getOrCreateTenantActor(tenant.getId()); + log.debug("[{}] Tenant actor created.", tenant.getId()); + } } } log.info("Main system actor started."); @@ -133,40 +140,33 @@ public class AppActor extends RuleChainManagerActor { } } - private void onPossibleClusterMsg(SendToClusterMsg msg) { - Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); - if (address.isPresent()) { - systemContext.getRpcService().tell( - systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); - } else { - self().tell(msg.getMsg(), ActorRef.noSender()); - } - } - - private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { + private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { if (SYSTEM_TENANT.equals(msg.getTenantId())) { -// this may be a notification about system entities created. -// log.warn("[{}] Invalid service to rule engine msg called. System messages are not supported yet: {}", SYSTEM_TENANT, msg); + msg.getTbMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); + if (!deletedTenants.contains(msg.getTenantId())) { + getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); + } else { + msg.getTbMsg().getCallback().onSuccess(); + } } } - @Override protected void broadcast(Object msg) { - super.broadcast(msg); tenantActors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { ActorRef target = null; if (SYSTEM_TENANT.equals(msg.getTenantId())) { - target = getEntityActorRef(msg.getEntityId()); + log.warn("Message has system tenant id: {}", msg); } else { if (msg.getEntityId().getEntityType() == EntityType.TENANT && msg.getEvent() == ComponentLifecycleEvent.DELETED) { - log.debug("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); - ActorRef tenantActor = tenantActors.remove(new TenantId(msg.getEntityId().getId())); + log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); + TenantId tenantId = new TenantId(msg.getEntityId().getId()); + deletedTenants.add(tenantId); + ActorRef tenantActor = tenantActors.get(tenantId); if (tenantActor != null) { log.debug("[{}] Deleting tenant actor: {}", msg.getTenantId(), tenantActor); context().stop(tenantActor); @@ -183,16 +183,22 @@ public class AppActor extends RuleChainManagerActor { } private void onToDeviceActorMsg(TenantAwareMsg msg) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); + if (!deletedTenants.contains(msg.getTenantId())) { + getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); + } else { + if (msg instanceof TransportToDeviceActorMsgWrapper) { + ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); + } + } } private ActorRef getOrCreateTenantActor(TenantId tenantId) { return tenantActors.computeIfAbsent(tenantId, k -> { - log.debug("[{}] Creating tenant actor.", tenantId); + log.info("[{}] Creating tenant actor.", tenantId); ActorRef tenantActor = context().actorOf(Props.create(new TenantActor.ActorCreator(systemContext, tenantId)) .withDispatcher(DefaultActorService.CORE_DISPATCHER_NAME), tenantId.toString()); context().watch(tenantActor); - log.debug("[{}] Created tenant actor: {}.", tenantId, tenantActor); + log.info("[{}] Created tenant actor: {}.", tenantId, tenantActor); return tenantActor; }); } diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java index 547e484652..256ffd0b30 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java @@ -22,10 +22,8 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; public class DeviceActor extends ContextAwareActor { @@ -48,6 +46,11 @@ public class DeviceActor extends ContextAwareActor { } } + @Override + public void postStop() { + + } + @Override protected boolean process(TbActorMsg msg) { switch (msg.getMsgType()) { @@ -66,15 +69,9 @@ public class DeviceActor extends ContextAwareActor { case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: processor.processRpcRequest(context(), (ToDeviceRpcRequestActorMsg) msg); break; - case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: - processor.processToServerRPCResponse(context(), (ToServerRpcResponseActorMsg) msg); - break; case DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG: processor.processServerSideRpcTimeout(context(), (DeviceActorServerSideRpcTimeoutMsg) msg); break; - case DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG: - processor.processClientSideRpcTimeout(context(), (DeviceActorClientSideRpcTimeoutMsg) msg); - break; case SESSION_TIMEOUT_MSG: processor.checkSessionsTimeout(); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 2752b53090..437ed456e9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -16,13 +16,10 @@ package org.thingsboard.server.actors.device; import akka.actor.ActorContext; -import com.datastax.driver.core.utils.UUIDs; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.Gson; -import com.google.gson.JsonObject; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -38,38 +35,34 @@ import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.session.SessionMsgType; -import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SessionSubscriptionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SessionType; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; -import org.thingsboard.server.utils.JsonUtils; import javax.annotation.Nullable; import java.util.ArrayList; @@ -100,9 +93,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private final Map attributeSubscriptions; private final Map rpcSubscriptions; private final Map toDeviceRpcPendingMap; - private final Map toServerRpcPendingMap; - - private final Gson gson = new Gson(); private int rpcSeq = 0; private String deviceName; @@ -117,7 +107,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { this.attributeSubscriptions = new HashMap<>(); this.rpcSubscriptions = new HashMap<>(); this.toDeviceRpcPendingMap = new HashMap<>(); - this.toServerRpcPendingMap = new HashMap<>(); if (initAttributes()) { restoreSessions(); } @@ -153,7 +142,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { Set syncSessionSet = new HashSet<>(); rpcSubscriptions.forEach((key, value) -> { sendToTransport(rpcRequest, key, value.getNodeId()); - if (TransportProtos.SessionType.SYNC == value.getType()) { + if (SessionType.SYNC == value.getType()) { syncSessionSet.add(key); } }); @@ -161,7 +150,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (request.isOneway() && sent) { log.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); } else { registerPendingRpcRequest(context, msg, sent, rpcRequest, timeout); } @@ -182,16 +171,16 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(msg.getId()); if (requestMd != null) { log.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION)); } } private void sendPendingRequests(ActorContext context, UUID sessionId, SessionInfoProto sessionInfo) { - TransportProtos.SessionType sessionType = getSessionType(sessionId); + SessionType sessionType = getSessionType(sessionId); if (!toDeviceRpcPendingMap.isEmpty()) { log.debug("[{}] Pushing {} pending RPC messages to new async session [{}]", deviceId, toDeviceRpcPendingMap.size(), sessionId); - if (sessionType == TransportProtos.SessionType.SYNC) { + if (sessionType == SessionType.SYNC) { log.debug("[{}] Cleanup sync rpc session [{}]", deviceId, sessionId); rpcSubscriptions.remove(sessionId); } @@ -199,7 +188,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No pending RPC messages for new async session [{}]", deviceId, sessionId); } Set sentOneWayIds = new HashSet<>(); - if (sessionType == TransportProtos.SessionType.ASYNC) { + if (sessionType == SessionType.ASYNC) { toDeviceRpcPendingMap.entrySet().forEach(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); } else { toDeviceRpcPendingMap.entrySet().stream().findFirst().ifPresent(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); @@ -214,7 +203,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestBody body = request.getBody(); if (request.isOneway()) { sentOneWayIds.add(entry.getKey()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); } ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder().setRequestId( entry.getKey()).setMethodName(body.getMethod()).setParams(body.getParams()).build(); @@ -223,8 +212,8 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { - boolean reportDeviceActivity = false; TransportToDeviceActorMsg msg = wrapper.getMsg(); + TbCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { processSessionStateMsgs(msg.getSessionInfo(), msg.getSessionEvent()); } @@ -234,34 +223,16 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (msg.hasSubscribeToRPC()) { processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToRPC()); } - if (msg.hasPostAttributes()) { - handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes()); - reportDeviceActivity = true; - } - if (msg.hasPostTelemetry()) { - handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry()); - reportDeviceActivity = true; - } if (msg.hasGetAttributes()) { handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes()); } if (msg.hasToDeviceRPCCallResponse()) { processRpcResponses(context, msg.getSessionInfo(), msg.getToDeviceRPCCallResponse()); } - if (msg.hasToServerRPCCallRequest()) { - handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest()); - reportDeviceActivity = true; - } if (msg.hasSubscriptionInfo()) { handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo()); } - if (reportDeviceActivity) { - reportLogicalDeviceActivity(); - } - } - - private void reportLogicalDeviceActivity() { - systemContext.getDeviceStateService().onDeviceActivity(deviceId); + callback.onSuccess(); } private void reportSessionOpen() { @@ -326,67 +297,8 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { return new HashSet<>(strings); } - private void handlePostAttributesRequest(ActorContext context, SessionInfoProto sessionInfo, PostAttributeMsg postAttributes) { - JsonObject json = JsonUtils.getJsonObject(postAttributes.getKvList()); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, defaultMetaData.copy(), - TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - pushToRuleEngine(context, tbMsg); - } - - private void handlePostTelemetryRequest(ActorContext context, SessionInfoProto sessionInfo, PostTelemetryMsg postTelemetry) { - for (TsKvListProto tsKv : postTelemetry.getTsKvListList()) { - JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); - TbMsgMetaData metaData = defaultMetaData.copy(); - metaData.putValue("ts", tsKv.getTs() + ""); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - pushToRuleEngine(context, tbMsg); - } - } - - private void handleClientSideRPCRequest(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg request) { - UUID sessionId = getSessionId(sessionInfo); - JsonObject json = new JsonObject(); - json.addProperty("method", request.getMethodName()); - json.add("params", JsonUtils.parse(request.getParams())); - - TbMsgMetaData requestMetaData = defaultMetaData.copy(); - requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); - - scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); - toServerRpcPendingMap.put(request.getRequestId(), new ToServerRpcRequestMetadata(sessionId, getSessionType(sessionId), sessionInfo.getNodeId())); - } - - private TransportProtos.SessionType getSessionType(UUID sessionId) { - return sessions.containsKey(sessionId) ? TransportProtos.SessionType.ASYNC : TransportProtos.SessionType.SYNC; - } - - void processClientSideRpcTimeout(ActorContext context, DeviceActorClientSideRpcTimeoutMsg msg) { - ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(msg.getId()); - if (data != null) { - log.debug("[{}] Client side RPC request [{}] timeout detected!", deviceId, msg.getId()); - sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(msg.getId()).setError("timeout").build() - , data.getSessionId(), data.getNodeId()); - } - } - - void processToServerRPCResponse(ActorContext context, ToServerRpcResponseActorMsg msg) { - int requestId = msg.getMsg().getRequestId(); - ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); - if (data != null) { - log.debug("[{}] Pushing reply to [{}][{}]!", deviceId, data.getNodeId(), data.getSessionId()); - sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(requestId).setPayload(msg.getMsg().getData()).build() - , data.getSessionId(), data.getNodeId()); - } else { - log.debug("[{}][{}] Pending RPC request to server not found!", deviceId, requestId); - } - } - - private void pushToRuleEngine(ActorContext context, TbMsg tbMsg) { - context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); + private SessionType getSessionType(UUID sessionId) { + return sessions.containsKey(sessionId) ? SessionType.ASYNC : SessionType.SYNC; } void processAttributesUpdate(ActorContext context, DeviceAttributesEventNotificationMsg msg) { @@ -434,7 +346,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(responseMsg.getRequestId()); boolean success = requestMd != null; if (success) { - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), responseMsg.getPayload(), null)); } else { log.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId()); @@ -449,7 +361,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToAttributes(true); log.debug("[{}] Registering attributes subscription for session [{}]", deviceId, sessionId); @@ -470,7 +382,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToRPC(true); log.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId); @@ -494,10 +406,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { notifyTransportAboutClosedSession(sessionIdToRemove, sessions.remove(sessionIdToRemove)); } } - sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfo.getNodeId()))); + sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfo.getNodeId()))); if (sessions.size() == 1) { reportSessionOpen(); } + systemContext.getDeviceStateService().onDeviceActivity(deviceId, System.currentTimeMillis()); dumpSessions(); } else if (msg.getEvent() == SessionEvent.CLOSED) { log.debug("[{}] Canceling subscriptions for closed session [{}]", deviceId, sessionId); @@ -511,10 +424,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, TransportProtos.SubscriptionInfoProto subscriptionInfo) { + private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, SubscriptionInfoProto subscriptionInfo) { UUID sessionId = getSessionId(sessionInfoProto); SessionInfoMetaData sessionMD = sessions.computeIfAbsent(sessionId, - id -> new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); + id -> new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); sessionMD.setLastActivityTime(subscriptionInfo.getLastActivityTime()); sessionMD.setSubscribedToAttributes(subscriptionInfo.getAttributeSubscription()); @@ -525,6 +438,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (subscriptionInfo.getRpcSubscription()) { rpcSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo()); } + systemContext.getDeviceStateService().onDeviceActivity(deviceId, subscriptionInfo.getLastActivityTime()); dumpSessions(); } @@ -536,11 +450,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void notifyTransportAboutClosedSession(UUID sessionId, SessionInfoMetaData sessionMd) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setSessionCloseNotification(SessionCloseNotificationProto.getDefaultInstance()).build(); - systemContext.getRuleEngineTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); + systemContext.getTbCoreToTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); } void processNameOrTypeUpdate(DeviceNameOrTypeUpdateMsg msg) { @@ -552,35 +466,35 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendToTransport(GetAttributeResponseMsg responseMsg, SessionInfoProto sessionInfo) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionInfo.getSessionIdMSB()) .setSessionIdLSB(sessionInfo.getSessionIdLSB()) .setGetAttributesResponse(responseMsg).build(); - systemContext.getRuleEngineTransportService().process(sessionInfo.getNodeId(), msg); + systemContext.getTbCoreToTransportService().process(sessionInfo.getNodeId(), msg); } private void sendToTransport(AttributeUpdateNotificationMsg notificationMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setAttributeUpdateNotification(notificationMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } private void sendToTransport(ToDeviceRpcRequestMsg rpcMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToDeviceRequest(rpcMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } - private void sendToTransport(TransportProtos.ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + private void sendToTransport(ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToServerResponse(rpcMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } @@ -632,9 +546,9 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void restoreSessions() { log.debug("[{}] Restoring sessions from cache", deviceId); - TransportProtos.DeviceSessionsCacheEntry sessionsDump = null; + DeviceSessionsCacheEntry sessionsDump = null; try { - sessionsDump = TransportProtos.DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); + sessionsDump = DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); } catch (InvalidProtocolBufferException e) { log.warn("[{}] Failed to decode device sessions from cache", deviceId); return; @@ -643,11 +557,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No session information found", deviceId); return; } - for (TransportProtos.SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { + for (SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { SessionInfoProto sessionInfoProto = sessionSubscriptionInfoProto.getSessionInfo(); UUID sessionId = getSessionId(sessionInfoProto); - SessionInfo sessionInfo = new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()); - TransportProtos.SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); + SessionInfo sessionInfo = new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()); + SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); SessionInfoMetaData sessionMD = new SessionInfoMetaData(sessionInfo, subInfo.getLastActivityTime()); sessions.put(sessionId, sessionMD); if (subInfo.getAttributeSubscription()) { @@ -665,27 +579,27 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void dumpSessions() { log.debug("[{}] Dumping sessions: {}, rpc subscriptions: {}, attribute subscriptions: {} to cache", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size()); - List sessionsList = new ArrayList<>(sessions.size()); + List sessionsList = new ArrayList<>(sessions.size()); sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getSessionInfo().getType() == TransportProtos.SessionType.SYNC) { + if (sessionMD.getSessionInfo().getType() == SessionType.SYNC) { return; } SessionInfo sessionInfo = sessionMD.getSessionInfo(); - TransportProtos.SubscriptionInfoProto subscriptionInfoProto = TransportProtos.SubscriptionInfoProto.newBuilder() + SubscriptionInfoProto subscriptionInfoProto = SubscriptionInfoProto.newBuilder() .setLastActivityTime(sessionMD.getLastActivityTime()) .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) .setRpcSubscription(sessionMD.isSubscribedToRPC()).build(); - TransportProtos.SessionInfoProto sessionInfoProto = TransportProtos.SessionInfoProto.newBuilder() + SessionInfoProto sessionInfoProto = SessionInfoProto.newBuilder() .setSessionIdMSB(uuid.getMostSignificantBits()) .setSessionIdLSB(uuid.getLeastSignificantBits()) .setNodeId(sessionInfo.getNodeId()).build(); - sessionsList.add(TransportProtos.SessionSubscriptionInfoProto.newBuilder() + sessionsList.add(SessionSubscriptionInfoProto.newBuilder() .setSessionInfo(sessionInfoProto) .setSubscriptionInfo(subscriptionInfoProto).build()); log.debug("[{}] Dumping session: {}", deviceId, sessionMD); }); systemContext.getDeviceSessionCacheService() - .put(deviceId, TransportProtos.DeviceSessionsCacheEntry.newBuilder() + .put(deviceId, DeviceSessionsCacheEntry.newBuilder() .addAllSessions(sessionsList).build().toByteArray()); } @@ -706,4 +620,5 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { dumpSessions(); } } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java b/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java deleted file mode 100644 index 8b502b973c..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; -import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class BasicRpcSessionListener implements GrpcSessionListener { - - private final ClusterRpcCallbackExecutorService callbackExecutorService; - private final ActorService service; - private final ActorRef manager; - private final ActorRef self; - - BasicRpcSessionListener(ActorSystemContext context, ActorRef manager, ActorRef self) { - this.service = context.getActorService(); - this.callbackExecutorService = context.getClusterRpcCallbackExecutor(); - this.manager = manager; - this.self = self; - } - - @Override - public void onConnected(GrpcSession session) { - log.info("[{}][{}] session started", session.getRemoteServer(), getType(session)); - if (!session.isClient()) { - manager.tell(new RpcSessionConnectedMsg(session.getRemoteServer(), session.getSessionId()), self); - } - } - - @Override - public void onDisconnected(GrpcSession session) { - log.info("[{}][{}] session closed", session.getRemoteServer(), getType(session)); - manager.tell(new RpcSessionDisconnectedMsg(session.isClient(), session.getRemoteServer()), self); - } - - @Override - public void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage) { - log.trace("Received session actor msg from [{}][{}]: {}", session.getRemoteServer(), getType(session), clusterMessage); - callbackExecutorService.execute(() -> { - try { - service.onReceivedMsg(session.getRemoteServer(), clusterMessage); - } catch (Exception e) { - log.debug("[{}][{}] Failed to process cluster message: {}", session.getRemoteServer(), getType(session), clusterMessage, e); - } - }); - } - - @Override - public void onError(GrpcSession session, Throwable t) { - log.warn("[{}][{}] session got error -> {}", session.getRemoteServer(), getType(session), t); - manager.tell(new RpcSessionClosedMsg(session.isClient(), session.getRemoteServer()), self); - session.close(); - } - - private static String getType(GrpcSession session) { - return session.isClient() ? "Client" : "Server"; - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java deleted file mode 100644 index 133e7375d4..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java +++ /dev/null @@ -1,230 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import akka.actor.OneForOneStrategy; -import akka.actor.Props; -import akka.actor.SupervisorStrategy; -import akka.event.Logging; -import akka.event.LoggingAdapter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import scala.concurrent.duration.Duration; - -import java.util.*; - -/** - * @author Andrew Shvayka - */ -public class RpcManagerActor extends ContextAwareActor { - - private final Map sessionActors; - private final Map> pendingMsgs; - private final ServerAddress instance; - - private RpcManagerActor(ActorSystemContext systemContext) { - super(systemContext); - this.sessionActors = new HashMap<>(); - this.pendingMsgs = new HashMap<>(); - this.instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - - systemContext.getDiscoveryService().getOtherServers().stream() - .filter(otherServer -> otherServer.getServerAddress().compareTo(instance) > 0) - .forEach(otherServer -> onCreateSessionRequest( - new RpcSessionCreateRequestMsg(UUID.randomUUID(), otherServer.getServerAddress(), null))); - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - onMsg((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcBroadcastMsg) { - onMsg((RpcBroadcastMsg) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - onCreateSessionRequest((RpcSessionCreateRequestMsg) msg); - } else if (msg instanceof RpcSessionConnectedMsg) { - onSessionConnected((RpcSessionConnectedMsg) msg); - } else if (msg instanceof RpcSessionDisconnectedMsg) { - onSessionDisconnected((RpcSessionDisconnectedMsg) msg); - } else if (msg instanceof RpcSessionClosedMsg) { - onSessionClosed((RpcSessionClosedMsg) msg); - } else if (msg instanceof ClusterEventMsg) { - onClusterEvent((ClusterEventMsg) msg); - } - } - - private void onMsg(RpcBroadcastMsg msg) { - log.debug("Forwarding msg to session actors {}", msg); - sessionActors.keySet().forEach(address -> { - ClusterAPIProtos.ClusterMessage msgWithServerAddress = msg.getMsg() - .toBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(address.getHost()) - .setPort(address.getPort()) - .build()) - .build(); - onMsg(msgWithServerAddress); - }); - pendingMsgs.values().forEach(queue -> queue.add(msg.getMsg())); - } - - private void onMsg(ClusterAPIProtos.ClusterMessage msg) { - if (msg.hasServerAddress()) { - ServerAddress address = new ServerAddress(msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE); - SessionActorInfo session = sessionActors.get(address); - if (session != null) { - log.debug("{} Forwarding msg to session actor: {}", address, msg); - session.getActor().tell(msg, ActorRef.noSender()); - } else { - log.debug("{} Storing msg to pending queue: {}", address, msg); - Queue queue = pendingMsgs.get(address); - if (queue == null) { - queue = new LinkedList<>(); - pendingMsgs.put(new ServerAddress( - msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE), queue); - } - queue.add(msg); - } - } else { - log.warn("Cluster msg doesn't have server address [{}]", msg); - } - } - - @Override - public void postStop() { - sessionActors.clear(); - pendingMsgs.clear(); - } - - private void onClusterEvent(ClusterEventMsg msg) { - ServerAddress server = msg.getServerAddress(); - if (server.compareTo(instance) > 0) { - if (msg.isAdded()) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(UUID.randomUUID(), server, null)); - } else { - onSessionClose(false, server); - } - } - } - - private void onSessionConnected(RpcSessionConnectedMsg msg) { - register(msg.getRemoteAddress(), msg.getId(), context().sender()); - } - - private void onSessionDisconnected(RpcSessionDisconnectedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private void onSessionClosed(RpcSessionClosedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private boolean isRegistered(ServerAddress address) { - for (ServerInstance server : systemContext.getDiscoveryService().getOtherServers()) { - if (server.getServerAddress().equals(address)) { - return true; - } - } - return false; - } - - private void onSessionClose(boolean reconnect, ServerAddress remoteAddress) { - log.info("[{}] session closed. Should reconnect: {}", remoteAddress, reconnect); - SessionActorInfo sessionRef = sessionActors.get(remoteAddress); - if (sessionRef != null && context().sender() != null && context().sender().equals(sessionRef.actor)) { - context().stop(sessionRef.actor); - sessionActors.remove(remoteAddress); - pendingMsgs.remove(remoteAddress); - if (reconnect) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(sessionRef.sessionId, remoteAddress, null)); - } - } - } - - private void onCreateSessionRequest(RpcSessionCreateRequestMsg msg) { - if (msg.getRemoteAddress() != null) { - if (!sessionActors.containsKey(msg.getRemoteAddress())) { - ActorRef actorRef = createSessionActor(msg); - register(msg.getRemoteAddress(), msg.getMsgUid(), actorRef); - } - } else { - createSessionActor(msg); - } - } - - private void register(ServerAddress remoteAddress, UUID uuid, ActorRef sender) { - sessionActors.put(remoteAddress, new SessionActorInfo(uuid, sender)); - log.info("[{}][{}] Registering session actor.", remoteAddress, uuid); - Queue data = pendingMsgs.remove(remoteAddress); - if (data != null) { - log.info("[{}][{}] Forwarding {} pending messages.", remoteAddress, uuid, data.size()); - data.forEach(msg -> sender.tell(new RpcSessionTellMsg(msg), ActorRef.noSender())); - } else { - log.info("[{}][{}] No pending messages to forward.", remoteAddress, uuid); - } - } - - private ActorRef createSessionActor(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Creating session actor.", msg.getMsgUid()); - ActorRef actor = context().actorOf( - Props.create(new RpcSessionActor.ActorCreator(systemContext, msg.getMsgUid())) - .withDispatcher(DefaultActorService.RPC_DISPATCHER_NAME)); - actor.tell(msg, context().self()); - return actor; - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - public ActorCreator(ActorSystemContext context) { - super(context); - } - - @Override - public RpcManagerActor create() { - return new RpcManagerActor(context); - } - } - - @Override - public SupervisorStrategy supervisorStrategy() { - return strategy; - } - - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { - log.warn("Unknown failure", t); - return SupervisorStrategy.resume(); - }); -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java deleted file mode 100644 index af2a7f0631..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; - -import java.util.UUID; - -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class RpcSessionActor extends ContextAwareActor { - - - private final UUID sessionId; - private GrpcSession session; - private GrpcSessionListener listener; - - private RpcSessionActor(ActorSystemContext systemContext, UUID sessionId) { - super(systemContext); - this.sessionId = sessionId; - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - tell((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - initSession((RpcSessionCreateRequestMsg) msg); - } - } - - private void tell(ClusterAPIProtos.ClusterMessage msg) { - if (session != null) { - session.sendMsg(msg); - } else { - log.trace("Failed to send message due to missing session!"); - } - } - - @Override - public void postStop() { - if (session != null) { - log.info("Closing session -> {}", session.getRemoteServer()); - try { - session.close(); - } catch (RuntimeException e) { - log.trace("Failed to close session!", e); - } - } - } - - private void initSession(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Initializing session", context().self()); - ServerAddress remoteServer = msg.getRemoteAddress(); - listener = new BasicRpcSessionListener(systemContext, context().parent(), context().self()); - if (msg.getRemoteAddress() == null) { - // Server session - session = new GrpcSession(listener); - session.setOutputStream(msg.getResponseObserver()); - session.initInputStream(); - session.initOutputStream(); - systemContext.getRpcService().onSessionCreated(msg.getMsgUid(), session.getInputStream()); - } else { - // Client session - ManagedChannel channel = ManagedChannelBuilder.forAddress(remoteServer.getHost(), remoteServer.getPort()).usePlaintext().build(); - session = new GrpcSession(remoteServer, listener, channel); - session.initInputStream(); - - ClusterRpcServiceGrpc.ClusterRpcServiceStub stub = ClusterRpcServiceGrpc.newStub(channel); - StreamObserver outputStream = stub.handleMsgs(session.getInputStream()); - - session.setOutputStream(outputStream); - session.initOutputStream(); - outputStream.onNext(toConnectMsg()); - } - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - private final UUID sessionId; - - public ActorCreator(ActorSystemContext context, UUID sessionId) { - super(context); - this.sessionId = sessionId; - } - - @Override - public RpcSessionActor create() { - return new RpcSessionActor(context, sessionId); - } - } - - private ClusterAPIProtos.ClusterMessage toConnectMsg() { - ServerAddress instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - return ClusterAPIProtos.ClusterMessage.newBuilder().setMessageType(CONNECT_RPC_MESSAGE).setServerAddress( - ClusterAPIProtos.ServerAddress.newBuilder().setHost(instance.getHost()) - .setPort(instance.getPort()).build()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java deleted file mode 100644 index 3832d6eb94..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionTellMsg { - private final ClusterAPIProtos.ClusterMessage msg; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index b700007498..6a3994a36f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -17,18 +17,13 @@ package org.thingsboard.server.actors.ruleChain; import akka.actor.ActorRef; import com.datastax.driver.core.ResultSetFuture; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.util.StringUtils; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -40,19 +35,15 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -67,11 +58,13 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; import java.util.Collections; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -79,6 +72,7 @@ import java.util.function.Consumer; /** * Created by ashvayka on 19.03.18. */ +@Slf4j class DefaultTbContext implements TbContext { public final static ObjectMapper mapper = new ObjectMapper(); @@ -91,6 +85,11 @@ class DefaultTbContext implements TbContext { this.nodeCtx = nodeCtx; } + @Override + public void tellSuccess(TbMsg msg) { + tellNext(msg, Collections.singleton(TbRelationTypes.SUCCESS), null); + } + @Override public void tellNext(TbMsg msg, String relationType) { tellNext(msg, Collections.singleton(relationType), null); @@ -101,16 +100,11 @@ class DefaultTbContext implements TbContext { tellNext(msg, relationTypes, null); } - @Override - public void tellNext(TbMsg msg, String relationType, Throwable th) { - tellNext(msg, Collections.singleton(relationType), th); - } - private void tellNext(TbMsg msg, Set relationTypes, Throwable th) { if (nodeCtx.getSelf().isDebugMode()) { relationTypes.forEach(relationType -> mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, relationType, th)); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); } @Override @@ -119,10 +113,94 @@ class DefaultTbContext implements TbContext { scheduleMsgWithDelay(new RuleNodeToSelfMsg(msg), delayMs, nodeCtx.getSelfActor()); } + @Override + public void enqueue(TbMsg tbMsg, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueue(tpi, tbMsg, onFailure, onSuccess); + } + + @Override + public void enqueue(TbMsg tbMsg, String queueName, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueue(tpi, tbMsg, onFailure, onSuccess); + } + + private void enqueue(TopicPartitionInfo tpi, TbMsg tbMsg, Consumer onFailure, Runnable onSuccess) { + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, new SimpleTbQueueCallback(onSuccess, onFailure)); + } + + @Override + public void enqueueForTellFailure(TbMsg tbMsg, String failureMessage) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(TbRelationTypes.FAILURE), failureMessage, null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String relationType) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String relationType, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); + } + + private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, String failureMessage, Runnable onSuccess, Consumer onFailure) { + RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId(); + RuleNodeId ruleNodeId = nodeCtx.getSelf().getId(); + tbMsg = TbMsg.newMsg(tbMsg, ruleChainId, ruleNodeId); + TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)) + .addAllRelationTypes(relationTypes); + if (failureMessage != null) { + msg.setFailureMessage(failureMessage); + } + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg.build(), new SimpleTbQueueCallback(onSuccess, onFailure)); + } + + @Override + public void ack(TbMsg tbMsg) { + if (nodeCtx.getSelf().isDebugMode()) { + mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "ACK", null); + } + tbMsg.getCallback().onSuccess(); + } + @Override public boolean isLocalEntity(EntityId entityId) { - Optional address = mainCtx.getRoutingService().resolveById(entityId); - return !address.isPresent(); + return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -134,66 +212,48 @@ class DefaultTbContext implements TbContext { if (nodeCtx.getSelf().isDebugMode()) { mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, TbRelationTypes.FAILURE, th); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), msg), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), + msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); } - @Override public void updateSelf(RuleNode self) { nodeCtx.setSelf(self); } @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), mainCtx.getQueuePartitionId()); + return TbMsg.newMsg(type, originator, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); } @Override public TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), mainCtx.getQueuePartitionId()); - } - - @Override - public void sendTbMsgToRuleEngine(TbMsg msg) { - mainCtx.getActorService().onMsg(new SendToClusterMsg(msg.getOriginator(), new ServiceToRuleEngineMsg(getTenantId(), msg))); + return TbMsg.transformMsg(origMsg, type, originator, metaData, data); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(customer); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, customer.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process customer created msg: " + e); - } + return entityCreatedMsg(customer, customer.getId(), ruleNodeId); } public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(device); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, device.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process device created msg: " + e); - } + return entityCreatedMsg(device, device.getId(), ruleNodeId); } public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(asset); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, asset.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process asset created msg: " + e); - } + return entityCreatedMsg(asset, asset.getId(), ruleNodeId); } public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) { + return entityCreatedMsg(alarm, alarm.getId(), ruleNodeId); + } + + public TbMsg entityCreatedMsg(E entity, I id, RuleNodeId ruleNodeId) { try { - ObjectNode entityNode = mapper.valueToTree(alarm); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, alarm.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + return TbMsg.newMsg(DataConstants.ENTITY_CREATED, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process alarm created msg: " + e); + throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " created msg: " + e); } } - @Override public RuleNodeId getSelfId() { return nodeCtx.getSelf().getId(); @@ -251,8 +311,8 @@ class DefaultTbContext implements TbContext { } @Override - public String getNodeId() { - return mainCtx.getNodeIdProvider().getNodeId(); + public String getServiceId() { + return mainCtx.getServiceInfoProvider().getServiceId(); } @Override @@ -320,11 +380,6 @@ class DefaultTbContext implements TbContext { return mainCtx.getEntityViewService(); } - @Override - public RuleChainTransactionService getRuleChainTransactionService() { - return mainCtx.getRuleChainTransactionService(); - } - @Override public EventLoopGroup getSharedEventLoop() { return mainCtx.getSharedEventLoopGroupService().getSharedEventLoopGroup(); @@ -341,35 +396,7 @@ class DefaultTbContext implements TbContext { @Override public RuleEngineRpcService getRpcService() { - return new RuleEngineRpcService() { - @Override - public void sendRpcReply(DeviceId deviceId, int requestId, String body) { - mainCtx.getDeviceRpcService().sendReplyToRpcCallFromDevice(nodeCtx.getTenantId(), deviceId, requestId, body); - } - - @Override - public void sendRpcRequest(RuleEngineDeviceRpcRequest src, Consumer consumer) { - ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), nodeCtx.getTenantId(), src.getDeviceId(), - src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); - mainCtx.getDeviceRpcService().forwardServerSideRPCRequestToDeviceActor(request, response -> { - if (src.isRestApiCall()) { - ServerAddress requestOriginAddress; - if (!StringUtils.isEmpty(src.getOriginHost())) { - requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); - } else { - requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); - } - mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); - } - consumer.accept(RuleEngineDeviceRpcResponse.builder() - .deviceId(src.getDeviceId()) - .requestId(src.getRequestId()) - .error(response.getError()) - .response(response.getResponse()) - .build()); - }); - } - }; + return mainCtx.getTbRuleEngineDeviceRpcService(); } @Override @@ -387,10 +414,6 @@ class DefaultTbContext implements TbContext { return mainCtx.getRedisTemplate(); } - @Override - public String getServerAddress() { - return mainCtx.getServerAddress(); - } private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) { TbMsgMetaData metaData = new TbMsgMetaData(); @@ -398,4 +421,29 @@ class DefaultTbContext implements TbContext { return metaData; } + private class SimpleTbQueueCallback implements TbQueueCallback { + private final Runnable onSuccess; + private final Consumer onFailure; + + public SimpleTbQueueCallback(Runnable onSuccess, Consumer onFailure) { + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (onSuccess != null) { + onSuccess.run(); + } + } + + @Override + public void onFailure(Throwable t) { + if (onFailure != null) { + onFailure.accept(t); + } else { + log.debug("[{}] Failed to put item into queue", nodeCtx.getTenantId(), t); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java deleted file mode 100644 index d062c5dc49..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.ruleChain; - -import lombok.Data; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; -import org.thingsboard.server.common.msg.aware.TenantAwareMsg; - -import java.io.Serializable; - -/** - * Created by ashvayka on 19.03.18. - */ -@Data -final class RemoteToRuleChainTellNextMsg extends RuleNodeToRuleChainTellNextMsg implements TenantAwareMsg, RuleChainAwareMsg { - - private static final long serialVersionUID = 2459605482321657447L; - private final TenantId tenantId; - private final RuleChainId ruleChainId; - - public RemoteToRuleChainTellNextMsg(RuleNodeToRuleChainTellNextMsg original, TenantId tenantId, RuleChainId ruleChainId) { - super(original.getOriginator(), original.getRelationTypes(), original.getMsg()); - this.tenantId = tenantId; - this.ruleChainId = ruleChainId; - } - - @Override - public MsgType getMsgType() { - return MsgType.REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 72faaef533..027f4aac9d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -15,25 +15,25 @@ */ package org.thingsboard.server.actors.ruleChain; -import akka.actor.ActorInitializationException; import akka.actor.OneForOneStrategy; import akka.actor.SupervisorStrategy; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.service.ComponentActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import scala.concurrent.duration.Duration; public class RuleChainActor extends ComponentActor { - private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId) { - super(systemContext, tenantId, ruleChainId); - setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChainId, systemContext, + private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChain ruleChain) { + super(systemContext, tenantId, ruleChain.getId()); + setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChain, systemContext, context().parent(), context().self())); } @@ -43,20 +43,17 @@ public class RuleChainActor extends ComponentActor { - private static final long DEFAULT_CLUSTER_PARTITION = 0L; private final ActorRef parent; private final ActorRef self; private final Map nodeActors; private final Map> nodeRoutes; private final RuleChainService service; + private final TbClusterService clusterService; + private String ruleChainName; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; - private String ruleChainName; - RuleChainActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, ActorSystemContext systemContext + RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { - super(systemContext, tenantId, ruleChainId); + super(systemContext, tenantId, ruleChain.getId()); + this.ruleChainName = ruleChain.getName(); this.parent = parent; this.self = self; this.nodeActors = new HashMap<>(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.ruleChainName = ruleChainId.toString(); + this.clusterService = systemContext.getClusterService(); } @Override @@ -92,7 +96,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor ruleNodeList = service.getRuleChainNodes(tenantId, entityId); log.trace("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); // Creating and starting the actors; @@ -152,8 +155,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actorRef.tell(msg, self)); } private ActorRef createRuleNodeActor(ActorContext context, RuleNode ruleNode) { @@ -192,100 +195,123 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor address = systemContext.getRoutingService().resolveById(originatorEntityId); + onTellNext(envelope.getMsg(), envelope.getOriginator(), envelope.getRelationTypes(), envelope.getFailureMessage()); + } - if (address.isPresent()) { - onRemoteTellNext(address.get(), envelope); - } else { - onLocalTellNext(envelope); + private void onTellNext(TbMsg msg, RuleNodeId originatorNodeId, Set relationTypes, String failureMessage) { + try { + checkActive(); + EntityId entityId = msg.getOriginator(); + TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + List relations = nodeRoutes.get(originatorNodeId).stream() + .filter(r -> contains(relationTypes, r.getType())) + .collect(Collectors.toList()); + int relationsCount = relations.size(); + if (relationsCount == 0) { + log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); + if (relationTypes.contains(TbRelationTypes.FAILURE)) { + RuleNodeCtx ruleNodeCtx = nodeActors.get(originatorNodeId); + if (ruleNodeCtx != null) { + msg.getCallback().onFailure(new RuleNodeException(failureMessage, ruleChainName, ruleNodeCtx.getSelf())); + } else { + log.debug("[{}] Failure during message processing by Rule Node [{}]. Enable and see debug events for more info", entityId, originatorNodeId.getId()); + msg.getCallback().onFailure(new RuleEngineException("Failure during message processing by Rule Node [" + originatorNodeId.getId().toString() + "]")); + } + } else { + msg.getCallback().onSuccess(); + } + } else if (relationsCount == 1) { + for (RuleNodeRelation relation : relations) { + log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); + pushToTarget(tpi, msg, relation.getOut(), relation.getType()); + } + } else { + MultipleTbQueueTbMsgCallbackWrapper callbackWrapper = new MultipleTbQueueTbMsgCallbackWrapper(relationsCount, msg.getCallback()); + log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relations); + for (RuleNodeRelation relation : relations) { + EntityId target = relation.getOut(); + putToQueue(tpi, msg, callbackWrapper, target); + } + } + } catch (Exception e) { + msg.getCallback().onFailure(new RuleEngineException("onTellNext - " + e.getMessage())); } } - private void onRemoteTellNext(ServerAddress serverAddress, RuleNodeToRuleChainTellNextMsg envelope) { - TbMsg msg = envelope.getMsg(); - log.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator()); - envelope = new RemoteToRuleChainTellNextMsg(envelope, tenantId, entityId); - systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); + private void putToQueue(TopicPartitionInfo tpi, TbMsg msg, TbQueueCallback callbackWrapper, EntityId target) { + switch (target.getEntityType()) { + case RULE_NODE: + putToQueue(tpi, msg.copyWithRuleNodeId(entityId, new RuleNodeId(target.getId())), callbackWrapper); + break; + case RULE_CHAIN: + putToQueue(tpi, msg.copyWithRuleChainId(new RuleChainId(target.getId())), callbackWrapper); + break; + } } - private void onLocalTellNext(RuleNodeToRuleChainTellNextMsg envelope) { - TbMsg msg = envelope.getMsg(); - RuleNodeId originatorNodeId = envelope.getOriginator(); - List relations = nodeRoutes.get(originatorNodeId).stream() - .filter(r -> contains(envelope.getRelationTypes(), r.getType())) - .collect(Collectors.toList()); - int relationsCount = relations.size(); - EntityId ackId = msg.getRuleNodeId() != null ? msg.getRuleNodeId() : msg.getRuleChainId(); - if (relationsCount == 0) { - log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); - if (ackId != null) { -// TODO: Ack this message in Kafka -// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); - } - } else if (relationsCount == 1) { - for (RuleNodeRelation relation : relations) { - log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - pushToTarget(msg, relation.getOut(), relation.getType()); + private void pushToTarget(TopicPartitionInfo tpi, TbMsg msg, EntityId target, String fromRelationType) { + if (tpi.isMyPartition()) { + switch (target.getEntityType()) { + case RULE_NODE: + pushMsgToNode(nodeActors.get(new RuleNodeId(target.getId())), msg, fromRelationType); + break; + case RULE_CHAIN: + parent.tell(new RuleChainToRuleChainMsg(new RuleChainId(target.getId()), entityId, msg, fromRelationType), self); + break; } } else { - for (RuleNodeRelation relation : relations) { - EntityId target = relation.getOut(); - log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - switch (target.getEntityType()) { - case RULE_NODE: - enqueueAndForwardMsgCopyToNode(msg, target, relation.getType()); - break; - case RULE_CHAIN: - enqueueAndForwardMsgCopyToChain(msg, target, relation.getType()); - break; - } - } - //TODO: Ideally this should happen in async way when all targets confirm that the copied messages are successfully written to corresponding target queues. - if (ackId != null) { -// TODO: Ack this message in Kafka -// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); - } + putToQueue(tpi, msg, new TbQueueTbMsgCallbackWrapper(msg.getCallback()), target); } } + private void putToQueue(TopicPartitionInfo tpi, TbMsg newMsg, TbQueueCallback callbackWrapper) { + ToRuleEngineMsg toQueueMsg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(newMsg)) + .build(); + clusterService.pushMsgToRuleEngine(tpi, newMsg.getId(), toQueueMsg, callbackWrapper); + } + private boolean contains(Set relationTypes, String type) { if (relationTypes == null) { return true; @@ -298,38 +324,13 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actors; + @Getter + protected RuleChain rootChain; + @Getter + protected ActorRef rootChainActor; - public RuleChainManagerActor(ActorSystemContext systemContext, RuleChainManager ruleChainManager) { + public RuleChainManagerActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); - this.ruleChainManager = ruleChainManager; + this.tenantId = tenantId; + this.actors = HashBiMap.create(); this.ruleChainService = systemContext.getRuleChainService(); } protected void initRuleChains() { - ruleChainManager.init(this.context()); + for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) { + RuleChainId ruleChainId = ruleChain.getId(); + log.debug("[{}|{}] Creating rule chain actor", ruleChainId.getEntityType(), ruleChain.getId()); + //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. + ActorRef actorRef = getOrCreateActor(this.context(), ruleChainId, id -> ruleChain); + visit(ruleChain, actorRef); + log.debug("[{}|{}] Rule Chain actor created.", ruleChainId.getEntityType(), ruleChainId.getId()); + } + } + + protected void visit(RuleChain entity, ActorRef actorRef) { + if (entity != null && entity.isRoot()) { + rootChain = entity; + rootChainActor = actorRef; + } + } + + public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId) { + return getOrCreateActor(context, ruleChainId, eId -> ruleChainService.findRuleChainById(TenantId.SYS_TENANT_ID, eId)); + } + + public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId, Function provider) { + return actors.computeIfAbsent(ruleChainId, eId -> { + RuleChain ruleChain = provider.apply(eId); + return context.actorOf(Props.create(new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain)) + .withDispatcher(DefaultActorService.TENANT_RULE_DISPATCHER_NAME), eId.toString()); + }); } protected ActorRef getEntityActorRef(EntityId entityId) { ActorRef target = null; - switch (entityId.getEntityType()) { - case RULE_CHAIN: - target = ruleChainManager.getOrCreateActor(this.context(), (RuleChainId) entityId); - break; + if (entityId.getEntityType() == EntityType.RULE_CHAIN) { + target = getOrCreateActor(this.context(), (RuleChainId) entityId); } return target; } protected void broadcast(Object msg) { - ruleChainManager.broadcast(msg); + actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } + + public ActorRef get(RuleChainId id) { + return actors.get(id); + } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java index e081768d93..0122471c52 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainToRuleChainMsg.java @@ -32,7 +32,6 @@ public final class RuleChainToRuleChainMsg implements TbActorMsg, RuleChainAware private final RuleChainId source; private final TbMsg msg; private final String fromRelationType; - private final boolean enqueue; @Override public RuleChainId getRuleChainId() { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java index 65227818e8..abb98468bf 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; public class RuleNodeActor extends ComponentActor { @@ -53,6 +54,9 @@ public class RuleNodeActor extends ComponentActor relationTypes; private final TbMsg msg; + private final String failureMessage; @Override public MsgType getMsgType() { diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index 3032b1dfe0..f6403c5d56 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -15,24 +15,7 @@ */ package org.thingsboard.server.actors.service; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; -import org.thingsboard.server.common.transport.SessionMsgProcessor; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.rpc.RpcMsgListener; +public interface ActorService { -public interface ActorService extends SessionMsgProcessor, RpcMsgListener, DiscoveryServiceListener { - - void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); - - void onMsg(SendToClusterMsg msg); - - void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId); - - void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType); } diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java b/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java index 3260c7ee31..2a92caf4f6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ComponentActor.java @@ -22,7 +22,7 @@ import org.thingsboard.server.actors.stats.StatsPersistMsg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; /** @@ -115,9 +115,9 @@ public abstract class ComponentActor status = system.terminate(); @@ -132,157 +97,4 @@ public class DefaultActorService implements ActorService { } } - @Override - public void onMsg(SendToClusterMsg msg) { - appActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onServerAdded(ServerInstance server) { - log.trace("Processing onServerAdded msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), true)); - } - - @Override - public void onServerUpdated(ServerInstance server) { - //Do nothing - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.trace("Processing onServerRemoved msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), false)); - } - - @Override - public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { - log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); - broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); - } - - @Override - public void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId) { - DeviceCredentialsUpdateNotificationMsg msg = new DeviceCredentialsUpdateNotificationMsg(tenantId, deviceId); - appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); - } - - @Override - public void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType) { - log.trace("[{}] Processing onDeviceNameOrTypeUpdate event, deviceName: {}, deviceType: {}", deviceId, deviceName, deviceType); - DeviceNameOrTypeUpdateMsg msg = new DeviceNameOrTypeUpdateMsg(tenantId, deviceId, deviceName, deviceType); - appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); - } - - public void broadcast(ToAllNodesMsg msg) { - actorContext.getEncodingService().encode(msg); - rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage - .newBuilder() - .setPayload(ByteString - .copyFrom(actorContext.getEncodingService().encode(msg))) - .setMessageType(CLUSTER_ACTOR_MESSAGE) - .build())); - appActor.tell(msg, ActorRef.noSender()); - } - - private void broadcast(ClusterEventMsg msg) { - this.appActor.tell(msg, ActorRef.noSender()); - this.rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Value("${cluster.stats.enabled:false}") - private boolean statsEnabled; - - private final AtomicInteger sentClusterMsgs = new AtomicInteger(0); - private final AtomicInteger receivedClusterMsgs = new AtomicInteger(0); - - - @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - int sent = sentClusterMsgs.getAndSet(0); - int received = receivedClusterMsgs.getAndSet(0); - if (sent > 0 || received > 0) { - log.info("Cluster msgs sent [{}] received [{}]", sent, received); - } - } - } - - @Override - public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - receivedClusterMsgs.incrementAndGet(); - } - ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); - if (log.isDebugEnabled()) { - log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); - log.info("MSG: {}", msg); - } - switch (msg.getMessageType()) { - case CLUSTER_ACTOR_MESSAGE: - java.util.Optional decodedMsg = actorContext.getEncodingService() - .decode(msg.getPayload().toByteArray()); - if (decodedMsg.isPresent()) { - appActor.tell(decodedMsg.get(), ActorRef.noSender()); - } else { - log.error("Error during decoding cluster proto message"); - } - break; - case TO_ALL_NODES_MSG: - //TODO - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: - actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: - actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: - actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TRANSACTION_SERVICE_MESSAGE: - actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); - break; - } - } - - @Override - public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onBroadcastMsg(RpcBroadcastMsg msg) { - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onDeviceAdded(Device device) { - deviceStateService.onDeviceAdded(device); - } } diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java index 2fdc918f8e..1443c61e16 100644 --- a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java @@ -16,20 +16,13 @@ package org.thingsboard.server.actors.shared; import akka.actor.ActorContext; -import akka.event.LoggingAdapter; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.stats.StatsPersistTick; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; - -import javax.annotation.Nullable; -import java.util.function.Consumer; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; @Slf4j public abstract class ComponentMsgProcessor extends AbstractContextAwareMsgProcessor { @@ -50,7 +43,7 @@ public abstract class ComponentMsgProcessor extends Abstract public abstract void stop(ActorContext context) throws Exception; - public abstract void onClusterEventMsg(ClusterEventMsg msg) throws Exception; + public abstract void onPartitionChangeMsg(PartitionChangeMsg msg) throws Exception; public void onCreated(ActorContext context) throws Exception { start(context); diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java deleted file mode 100644 index b824cf61ff..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared; - -import akka.actor.ActorContext; -import akka.actor.ActorRef; -import akka.actor.Props; -import akka.actor.UntypedActor; -import akka.japi.Creator; -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.common.data.SearchTextBased; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; -import org.thingsboard.server.common.data.page.PageDataIterable; - -import java.util.HashMap; -import java.util.Map; - -/** - * Created by ashvayka on 15.03.18. - */ -@Slf4j -public abstract class EntityActorsManager> { - - protected final ActorSystemContext systemContext; - protected final BiMap actors; - - public EntityActorsManager(ActorSystemContext systemContext) { - this.systemContext = systemContext; - this.actors = HashBiMap.create(); - } - - protected abstract TenantId getTenantId(); - - protected abstract String getDispatcherName(); - - protected abstract Creator creator(T entityId); - - protected abstract PageDataIterable.FetchFunction getFetchEntitiesFunction(); - - public void init(ActorContext context) { - for (M entity : new PageDataIterable<>(getFetchEntitiesFunction(), ContextAwareActor.ENTITY_PACK_LIMIT)) { - T entityId = (T) entity.getId(); - log.debug("[{}|{}] Creating entity actor", entityId.getEntityType(), entityId.getId()); - //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. - ActorRef actorRef = getOrCreateActor(context, entityId); - visit(entity, actorRef); - log.debug("[{}|{}] Entity actor created.", entityId.getEntityType(), entityId.getId()); - } - } - - public void visit(M entity, ActorRef actorRef) { - } - - public ActorRef getOrCreateActor(ActorContext context, T entityId) { - return actors.computeIfAbsent(entityId, eId -> - context.actorOf(Props.create(creator(eId)) - .withDispatcher(getDispatcherName()), eId.toString())); - } - - public void broadcast(Object msg) { - actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); - } - - public void remove(T id) { - actors.remove(id); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java deleted file mode 100644 index ef2b4282d4..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import akka.actor.ActorRef; -import akka.japi.Creator; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainActor; -import org.thingsboard.server.actors.shared.EntityActorsManager; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.dao.rule.RuleChainService; - -/** - * Created by ashvayka on 15.03.18. - */ -@Slf4j -public abstract class RuleChainManager extends EntityActorsManager { - - protected final RuleChainService service; - @Getter - protected RuleChain rootChain; - @Getter - protected ActorRef rootChainActor; - - public RuleChainManager(ActorSystemContext systemContext) { - super(systemContext); - this.service = systemContext.getRuleChainService(); - } - - @Override - public Creator creator(RuleChainId entityId) { - return new RuleChainActor.ActorCreator(systemContext, getTenantId(), entityId); - } - - @Override - public void visit(RuleChain entity, ActorRef actorRef) { - if (entity != null && entity.isRoot()) { - rootChain = entity; - rootChainActor = actorRef; - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java deleted file mode 100644 index f2a80a9e82..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; -import org.thingsboard.server.common.data.page.TextPageData; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.dao.model.ModelConstants; - -import java.util.Collections; - -public class SystemRuleChainManager extends RuleChainManager { - - public SystemRuleChainManager(ActorSystemContext systemContext) { - super(systemContext); - } - - @Override - protected FetchFunction getFetchEntitiesFunction() { - return link -> new TextPageData<>(Collections.emptyList(), link); - } - - @Override - protected TenantId getTenantId() { - return ModelConstants.SYSTEM_TENANT; - } - - @Override - protected String getDispatcherName() { - return DefaultActorService.SYSTEM_RULE_DISPATCHER_NAME; - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java deleted file mode 100644 index cfe862af49..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import akka.actor.ActorContext; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; -import org.thingsboard.server.common.data.rule.RuleChain; - -public class TenantRuleChainManager extends RuleChainManager { - - private final TenantId tenantId; - - public TenantRuleChainManager(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext); - this.tenantId = tenantId; - } - - @Override - public void init(ActorContext context) { - super.init(context); - } - - @Override - protected TenantId getTenantId() { - return tenantId; - } - - @Override - protected String getDispatcherName() { - return DefaultActorService.TENANT_RULE_DISPATCHER_NAME; - } - - @Override - protected FetchFunction getFetchEntitiesFunction() { - return link -> service.findTenantRuleChains(tenantId, link); - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 31461a8fac..717dcecc19 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -24,7 +24,6 @@ import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; @Slf4j public class StatsActor extends ContextAwareActor { @@ -58,12 +57,12 @@ public class StatsActor extends ContextAwareActor { event.setEntityId(msg.getEntityId()); event.setTenantId(msg.getTenantId()); event.setType(DataConstants.STATS); - event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); + event.setBody(toBodyJson(systemContext.getServiceInfoProvider().getServiceId(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); systemContext.getEventService().save(event); } - private JsonNode toBodyJson(ServerAddress server, long messagesProcessed, long errorsOccurred) { - return mapper.createObjectNode().put("server", server.toString()).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); + private JsonNode toBodyJson(String serviceId, long messagesProcessed, long errorsOccurred) { + return mapper.createObjectNode().put("server", serviceId).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); } public static class ActorCreator extends ContextBasedCreator { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 4aacfb55bf..3cee82127c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -22,41 +22,43 @@ import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; import akka.actor.Terminated; -import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.device.DeviceActorCreator; -import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.actors.shared.rulechain.TenantRuleChainManager; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceType; import scala.concurrent.duration.Duration; -import java.util.HashMap; -import java.util.Map; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; public class TenantActor extends RuleChainManagerActor { - private final TenantId tenantId; private final BiMap deviceActors; + private boolean isRuleEngineForCurrentTenant; + private boolean isCore; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext, new TenantRuleChainManager(systemContext, tenantId)); - this.tenantId = tenantId; + super(systemContext, tenantId); this.deviceActors = HashBiMap.create(); } @@ -65,12 +67,37 @@ public class TenantActor extends RuleChainManagerActor { return strategy; } + boolean cantFindTenant = false; + @Override public void preStart() { log.info("[{}] Starting tenant actor.", tenantId); try { - initRuleChains(); - log.info("[{}] Tenant actor started.", tenantId); + Tenant tenant = systemContext.getTenantService().findTenantById(tenantId); + if (tenant == null) { + cantFindTenant = true; + log.info("[{}] Started tenant actor for missing tenant.", tenantId); + } else { + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + + if (isRuleEngineForCurrentTenant) { + try { + if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { + log.info("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + isRuleEngineForCurrentTenant = false; + } + } catch (Exception e) { + cantFindTenant = true; + } + } + log.info("[{}] Tenant actor started.", tenantId); + } } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); } @@ -83,18 +110,36 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { + if (cantFindTenant) { + log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg); + if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { + QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; + queueMsg.getTbMsg().getCallback().onSuccess(); + } + return true; + } switch (msg.getMsgType()) { - case CLUSTER_EVENT_MSG: - broadcast(msg); + case PARTITION_CHANGE_MSG: + PartitionChangeMsg partitionChangeMsg = (PartitionChangeMsg) msg; + ServiceType serviceType = partitionChangeMsg.getServiceQueueKey().getServiceType(); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + //To Rule Chain Actors + broadcast(msg); + } else if (ServiceType.TB_CORE.equals(serviceType)) { + //To Device Actors + List repartitionedDevices = + deviceActors.keySet().stream().filter(deviceId -> !isMyPartition(deviceId)).collect(Collectors.toList()); + for (DeviceId deviceId : repartitionedDevices) { + ActorRef deviceActor = deviceActors.remove(deviceId); + context().stop(deviceActor); + } + } break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case SERVICE_TO_RULE_ENGINE_MSG: - onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); - break; - case DEVICE_ACTOR_TO_RULE_ENGINE_MSG: - onDeviceActorToRuleEngineMsg((DeviceActorToRuleEngineMsg) msg); + case QUEUE_TO_RULE_ENGINE_MSG: + onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -105,7 +150,6 @@ public class TenantActor extends RuleChainManagerActor { onToDeviceActorMsg((DeviceAwareMsg) msg); break; case RULE_CHAIN_TO_RULE_CHAIN_MSG: - case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; default: @@ -114,41 +158,59 @@ public class TenantActor extends RuleChainManagerActor { return true; } - private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { - if (ruleChainManager.getRootChainActor() != null) { - ruleChainManager.getRootChainActor().tell(msg, self()); - } else { - log.info("[{}] No Root Chain: {}", tenantId, msg); - } + private boolean isMyPartition(DeviceId deviceId) { + return systemContext.resolve(ServiceType.TB_CORE, tenantId, deviceId).isMyPartition(); } - private void onDeviceActorToRuleEngineMsg(DeviceActorToRuleEngineMsg msg) { - if (ruleChainManager.getRootChainActor() != null) { - ruleChainManager.getRootChainActor().tell(msg, self()); + private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { + if (!isRuleEngineForCurrentTenant) { + log.warn("RECEIVED INVALID MESSAGE: {}", msg); + return; + } + TbMsg tbMsg = msg.getTbMsg(); + if (tbMsg.getRuleChainId() == null) { + if (getRootChainActor() != null) { + getRootChainActor().tell(msg, self()); + } else { + tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); + log.info("[{}] No Root Chain: {}", tenantId, msg); + } } else { - log.info("[{}] No Root Chain: {}", tenantId, msg); + ActorRef ruleChainActor = get(tbMsg.getRuleChainId()); + if (ruleChainActor != null) { + ruleChainActor.tell(msg, self()); + } else { + log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId()); + //TODO: 3.1 Log it to dead letters queue; + tbMsg.getCallback().onSuccess(); + } } } private void onRuleChainMsg(RuleChainAwareMsg msg) { - ruleChainManager.getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); + getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); } private void onToDeviceActorMsg(DeviceAwareMsg msg) { + if (!isCore) { + log.warn("RECEIVED INVALID MESSAGE: {}", msg); + } getOrCreateDeviceActor(msg.getDeviceId()).tell(msg, ActorRef.noSender()); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { - ActorRef target = getEntityActorRef(msg.getEntityId()); - if (target != null) { - if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { - RuleChain ruleChain = systemContext.getRuleChainService(). - findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); - ruleChainManager.visit(ruleChain, target); + if (isRuleEngineForCurrentTenant) { + ActorRef target = getEntityActorRef(msg.getEntityId()); + if (target != null) { + if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { + RuleChain ruleChain = systemContext.getRuleChainService(). + findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); + visit(ruleChain, target); + } + target.tell(msg, ActorRef.noSender()); + } else { + log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } - target.tell(msg, ActorRef.noSender()); - } else { - log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } } @@ -172,7 +234,7 @@ public class TenantActor extends RuleChainManagerActor { if (removed) { log.debug("[{}] Removed actor:", terminated); } else { - log.warn("[{}] Removed actor was not found in the device map!"); + log.debug("Removed actor was not found in the device map!"); } } else { throw new IllegalStateException("Remote actors are not supported!"); @@ -195,15 +257,12 @@ public class TenantActor extends RuleChainManagerActor { } } - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), new Function() { - @Override - public SupervisorStrategy.Directive apply(Throwable t) { - log.warn("[{}] Unknown failure", tenantId, t); - if (t instanceof ActorInitializationException) { - return SupervisorStrategy.stop(); - } else { - return SupervisorStrategy.resume(); - } + private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { + log.warn("[{}] Unknown failure", tenantId, t); + if (t instanceof ActorInitializationException) { + return SupervisorStrategy.stop(); + } else { + return SupervisorStrategy.resume(); } }); diff --git a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java index 5e5bd94afa..e57695da69 100644 --- a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java @@ -32,11 +32,13 @@ import org.springframework.web.socket.server.support.HttpSessionHandshakeInterce import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.controller.plugin.TbWebSocketHandler; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; @Configuration +@TbCoreComponent @EnableWebSocket public class WebSocketConfiguration implements WebSocketConfigurer { @@ -62,7 +64,8 @@ public class WebSocketConfiguration implements WebSocketConfigurer { SecurityUser user = null; try { user = getCurrentUser(); - } catch (ThingsboardException ex) {} + } catch (ThingsboardException ex) { + } if (user == null) { response.setStatusCode(HttpStatus.UNAUTHORIZED); return false; diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index b7efc180a5..8873d82366 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -25,23 +25,25 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.common.data.security.model.SecuritySettings; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.update.UpdateService; -import org.thingsboard.server.common.data.UpdateMessage; @RestController +@TbCoreComponent @RequestMapping("/api/admin") public class AdminController extends BaseController { @Autowired private MailService mailService; - + @Autowired private AdminSettingsService adminSettingsService; @@ -65,7 +67,7 @@ public class AdminController extends BaseController { @PreAuthorize("hasAuthority('SYS_ADMIN')") @RequestMapping(value = "/settings", method = RequestMethod.POST) - @ResponseBody + @ResponseBody public AdminSettings saveAdminSettings(@RequestBody AdminSettings adminSettings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); @@ -111,8 +113,8 @@ public class AdminController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); adminSettings = checkNotNull(adminSettings); if (adminSettings.getKey().equals("mail")) { - String email = getCurrentUser().getEmail(); - mailService.sendTestMail(adminSettings.getJsonValue(), email); + String email = getCurrentUser().getEmail(); + mailService.sendTestMail(adminSettings.getJsonValue(), email); } } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 140c9ecf86..2132675f97 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; @@ -41,10 +41,12 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController +@TbCoreComponent @RequestMapping("/api") public class AlarmController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 0fbec9560f..03c5462446 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -32,16 +32,15 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; -import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -51,6 +50,7 @@ import java.util.List; import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class AssetController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java index 56f730f0fb..c99ba8a018 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.Arrays; import java.util.List; @@ -39,6 +40,7 @@ import java.util.UUID; import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class AuditLogController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 44449eb3d6..42da043a91 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; import org.thingsboard.server.common.data.security.model.SecuritySettings; @@ -56,6 +57,7 @@ import java.net.URI; import java.net.URISyntaxException; @RestController +@TbCoreComponent @RequestMapping("/api") @Slf4j public class AuthController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 634a78de27..8385e61a20 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -27,7 +26,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.DashboardInfo; @@ -36,10 +34,11 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; @@ -71,8 +70,6 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -93,7 +90,11 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.component.ComponentDiscoveryService; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; @@ -112,6 +113,7 @@ import java.util.UUID; import static org.thingsboard.server.dao.service.Validator.validateId; @Slf4j +@TbCoreComponent public abstract class BaseController { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; @@ -162,7 +164,7 @@ public abstract class BaseController { protected RuleChainService ruleChainService; @Autowired - protected ActorService actorService; + protected TbClusterService tbClusterService; @Autowired protected RelationService relationService; @@ -185,6 +187,12 @@ public abstract class BaseController { @Autowired protected ClaimDevicesService claimDevicesService; + @Autowired + protected PartitionService partitionService; + + @Autowired + protected TbQueueProducerProvider producerProvider; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -659,10 +667,14 @@ public abstract class BaseController { } } } - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, entityId, metaData, TbMsgDataType.JSON - , json.writeValueAsString(entityNode) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(entityId, new ServiceToRuleEngineMsg(user.getTenantId(), tbMsg))); + TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); + TenantId tenantId = user.getTenantId(); + if (tenantId.isNullUid()) { + if (entity instanceof HasTenantId) { + tenantId = ((HasTenantId) entity).getTenantId(); + } + } + tbClusterService.pushMsgToRuleEngine(tenantId, entityId, tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java index ef9508ad7c..d75da8b34c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java +++ b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java @@ -25,12 +25,14 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.HashSet; import java.util.List; import java.util.Set; @RestController +@TbCoreComponent @RequestMapping("/api") public class ComponentDescriptorController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index def876e272..18fcfc2ab5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -36,10 +36,12 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController +@TbCoreComponent @RequestMapping("/api") public class CustomerController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index 741a9fec95..68f18c9081 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; @@ -41,6 +40,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -48,6 +48,7 @@ import java.util.HashSet; import java.util.Set; @RestController +@TbCoreComponent @RequestMapping("/api") public class DashboardController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 23460506e9..ca61ecc8e9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -31,6 +31,8 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.data.ClaimRequest; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; @@ -50,6 +52,7 @@ import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -61,6 +64,7 @@ import java.util.List; import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class DeviceController extends BaseController { @@ -95,12 +99,8 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - actorService - .onDeviceNameOrTypeUpdate( - savedDevice.getTenantId(), - savedDevice.getId(), - savedDevice.getName(), - savedDevice.getType()); + tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), + savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); logEntityAction(savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), @@ -253,7 +253,9 @@ public class DeviceController extends BaseController { try { Device device = checkDeviceId(deviceCredentials.getDeviceId(), Operation.WRITE_CREDENTIALS); DeviceCredentials result = checkNotNull(deviceCredentialsService.updateDeviceCredentials(getCurrentUser().getTenantId(), deviceCredentials)); - actorService.onCredentialsUpdate(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()); + + tbClusterService.pushMsgToCore(new DeviceCredentialsUpdateNotificationMsg(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()), null); + logEntityAction(device.getId(), device, device.getCustomerId(), ActionType.CREDENTIALS_UPDATED, null, deviceCredentials); diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java index 8058cca685..88aa9ce850 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import java.util.List; @@ -40,6 +41,7 @@ import java.util.stream.Collectors; @RestController +@TbCoreComponent @RequestMapping("/api") public class EntityRelationController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java index 82053652d5..05c57d35fa 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java @@ -48,6 +48,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -65,6 +66,7 @@ import static org.thingsboard.server.controller.CustomerController.CUSTOMER_ID; * Created by Victor Basanets on 8/28/2017. */ @RestController +@TbCoreComponent @RequestMapping("/api") @Slf4j public class EntityViewController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EventController.java b/application/src/main/java/org/thingsboard/server/controller/EventController.java index d9ad9d54bf..500549c09a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EventController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EventController.java @@ -31,9 +31,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; @RestController +@TbCoreComponent @RequestMapping("/api") public class EventController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java new file mode 100644 index 0000000000..8d1cec7f4b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/QueueController.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +public class QueueController extends BaseController { + + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/tenant/queues", params = {"serviceType"}, method = RequestMethod.GET) + @ResponseBody + public List getTenantQueuesByServiceType(@RequestParam String serviceType) throws ThingsboardException { + checkParameter("serviceType", serviceType); + try { + ServiceType type = ServiceType.valueOf(serviceType); + switch (type) { + case TB_RULE_ENGINE: + return Arrays.asList("HighPriority", "Main"); + default: + return Collections.emptyList(); + } + } catch (Exception e) { + throw handleException(e); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index a8298ce82b..b88c3f3695 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -31,7 +31,6 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; @@ -43,27 +42,25 @@ import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.rpc.RpcRequest; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.service.rpc.DeviceRpcService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.LocalRequestMetaData; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.telemetry.exception.ToErrorResponseEntity; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.io.IOException; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; /** * Created by ashvayka on 22.03.18. */ @RestController +@TbCoreComponent @RequestMapping(TbUrlConstants.RPC_URL_PREFIX) @Slf4j public class RpcController extends BaseController { @@ -72,7 +69,7 @@ public class RpcController extends BaseController { protected final ObjectMapper jsonMapper = new ObjectMapper(); @Autowired - private DeviceRpcService deviceRpcService; + private TbCoreDeviceRpcService deviceRpcService; @Autowired private AccessValidator accessValidator; @@ -91,7 +88,6 @@ public class RpcController extends BaseController { return handleDeviceRPCRequest(false, new DeviceId(UUID.fromString(deviceIdStr)), requestBody); } - private DeferredResult handleDeviceRPCRequest(boolean oneWay, DeviceId deviceId, String requestBody) throws ThingsboardException { try { JsonNode rpcRequestBody = jsonMapper.readTree(requestBody); @@ -116,7 +112,7 @@ public class RpcController extends BaseController { timeout, body ); - deviceRpcService.processRestAPIRpcRequestToRuleEngine(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); + deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 5bfeeee523..5685947c9a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -52,8 +52,10 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import org.thingsboard.server.service.security.permission.Operation; @@ -67,6 +69,7 @@ import java.util.stream.Collectors; @Slf4j @RestController +@TbCoreComponent @RequestMapping("/api") public class RuleChainController extends BaseController { @@ -130,7 +133,7 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - actorService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); logEntityAction(savedRuleChain.getId(), savedRuleChain, @@ -161,7 +164,7 @@ public class RuleChainController extends BaseController { previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - actorService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), + tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, @@ -169,7 +172,7 @@ public class RuleChainController extends BaseController { ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, @@ -203,7 +206,7 @@ public class RuleChainController extends BaseController { RuleChain ruleChain = checkRuleChain(ruleChainMetaData.getRuleChainId(), Operation.WRITE); RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData)); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, null, @@ -254,9 +257,9 @@ public class RuleChainController extends BaseController { referencingRuleChainIds.remove(ruleChain.getId()); referencingRuleChainIds.forEach(referencingRuleChainId -> - actorService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); logEntityAction(ruleChainId, ruleChain, null, @@ -317,7 +320,7 @@ public class RuleChainController extends BaseController { ScriptEngine engine = null; try { engine = new RuleNodeJsScriptEngine(jsInvokeService, getCurrentUser().getId(), script, argNames); - TbMsg inMsg = new TbMsg(UUIDs.timeBased(), msgType, null, new TbMsgMetaData(metadata), data, null, null, 0L); + TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); switch (scriptType) { case "update": output = msgToOutput(engine.executeUpdate(inMsg)); diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index f607d57cf6..aee56652c4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -69,9 +69,9 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -99,6 +99,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 22.03.18. */ @RestController +@TbCoreComponent @RequestMapping(TbUrlConstants.TELEMETRY_URL_PREFIX) @Slf4j public class TelemetryController extends BaseController { @@ -343,7 +344,7 @@ public class TelemetryController extends BaseController { return deleteAttributes(entityId, scope, keysStr); } - private DeferredResult deleteAttributes(EntityId entityIdStr, String scope, String keysStr) throws ThingsboardException { + private DeferredResult deleteAttributes(EntityId entityIdSrc, String scope, String keysStr) throws ThingsboardException { List keys = toKeysList(keysStr); if (keys.isEmpty()) { return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST); @@ -353,19 +354,18 @@ public class TelemetryController extends BaseController { if (DataConstants.SERVER_SCOPE.equals(scope) || DataConstants.SHARED_SCOPE.equals(scope) || DataConstants.CLIENT_SCOPE.equals(scope)) { - return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdStr, (result, tenantId, entityId) -> { + return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { ListenableFuture> future = attributesService.removeAll(user.getTenantId(), entityId, scope, keys); Futures.addCallback(future, new FutureCallback>() { @Override public void onSuccess(@Nullable List tmp) { logAttributesDeleted(user, entityId, scope, keys, null); - if (entityId.getEntityType() == EntityType.DEVICE) { + if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onDelete( - user.getTenantId(), deviceId, keysToNotify); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete( + user.getTenantId(), deviceId, keysToNotify), null); } result.setResult(new ResponseEntity<>(HttpStatus.OK)); } @@ -397,11 +397,10 @@ public class TelemetryController extends BaseController { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); - if (entityId.getEntityType() == EntityType.DEVICE) { + if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { DeviceId deviceId = new DeviceId(entityId.getId()); - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate( - user.getTenantId(), deviceId, scope, attributes); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate( + user.getTenantId(), deviceId, scope, attributes), null); } result.setResult(new ResponseEntity(HttpStatus.OK)); } diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 8d95e78cfa..9def943e88 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -34,11 +34,13 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController +@TbCoreComponent @RequestMapping("/api") @Slf4j public class TenantController extends BaseController { @@ -74,7 +76,6 @@ public class TenantController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.TENANT, operation, tenant.getId(), tenant); - tenant = checkNotNull(tenantService.saveTenant(tenant)); if (newTenant) { installScripts.createDefaultRuleChains(tenant.getId()); @@ -94,8 +95,7 @@ public class TenantController extends BaseController { TenantId tenantId = new TenantId(toUUID(strTenantId)); checkTenantId(tenantId, Operation.DELETE); tenantService.deleteTenant(tenantId); - - actorService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); + tbClusterService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index a24efdb309..a37d9c29b6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -55,6 +56,7 @@ import org.thingsboard.server.service.security.permission.Resource; import javax.servlet.http.HttpServletRequest; @RestController +@TbCoreComponent @RequestMapping("/api") public class UserController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java index ce3e65c0a1..debe49b018 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java @@ -31,12 +31,14 @@ import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController +@TbCoreComponent @RequestMapping("/api") public class WidgetTypeController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java index b76306e071..3d5cd22400 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java @@ -32,13 +32,14 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController +@TbCoreComponent @RequestMapping("/api") public class WidgetsBundleController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index f2862c5ab6..6e33133522 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.config.WebSocketConfiguration; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.telemetry.SessionEvent; @@ -52,6 +53,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; @Service +@TbCoreComponent @Slf4j public class TbWebSocketHandler extends TextWebSocketHandler implements TelemetryWebSocketMsgEndpoint { @@ -63,7 +65,6 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr @Value("${server.ws.send_timeout:5000}") private long sendTimeout; - @Value("${server.ws.limits.max_sessions_per_tenant:0}") private int maxSessionsPerTenant; @Value("${server.ws.limits.max_sessions_per_customer:0}") @@ -91,10 +92,10 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr try { SessionMetaData sessionMd = internalSessionMap.get(session.getId()); if (sessionMd != null) { - log.info("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); + log.trace("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); webSocketService.handleWebSocketMsg(sessionMd.sessionRef, message.getPayload()); } else { - log.warn("[{}] Failed to find session", session.getId()); + log.trace("[{}] Failed to find session", session.getId()); session.close(CloseStatus.SERVER_ERROR.withReason("Session not found!")); } } catch (IOException e) { @@ -138,7 +139,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr if (sessionMd != null) { processInWebSocketService(sessionMd.sessionRef, SessionEvent.onError(tError)); } else { - log.warn("[{}] Failed to find session", session.getId()); + log.trace("[{}] Failed to find session", session.getId()); } log.trace("[{}] Session transport error", session.getId(), tError); } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java deleted file mode 100644 index 9a7c83f167..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; - -import javax.annotation.PostConstruct; - -import static org.thingsboard.server.utils.MiscUtils.missingProperty; - -/** - * @author Andrew Shvayka - */ -@Service -@Slf4j -public class CurrentServerInstanceService implements ServerInstanceService { - - @Value("${rpc.bind_host}") - private String rpcHost; - @Value("${rpc.bind_port}") - private Integer rpcPort; - - private ServerInstance self; - - @PostConstruct - public void init() { - Assert.hasLength(rpcHost, missingProperty("rpc.bind_host")); - Assert.notNull(rpcPort, missingProperty("rpc.bind_port")); - self = new ServerInstance(new ServerAddress(rpcHost, rpcPort, ServerType.CORE)); - log.info("Current server instance: [{};{}]", self.getHost(), self.getPort()); - } - - @Override - public ServerInstance getSelf() { - return self; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java deleted file mode 100644 index e9325800af..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@ToString -@EqualsAndHashCode(exclude = {"serverInfo", "serverAddress"}) -public final class ServerInstance implements Comparable { - - @Getter - private final String host; - @Getter - private final int port; - @Getter - private final ServerAddress serverAddress; - - public ServerInstance(ServerAddress serverAddress) { - this.serverAddress = serverAddress; - this.host = serverAddress.getHost(); - this.port = serverAddress.getPort(); - } - - @Override - public int compareTo(ServerInstance o) { - return this.serverAddress.compareTo(o.serverAddress); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java deleted file mode 100644 index e0ed64fdbd..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; - -import java.util.Optional; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -public interface ClusterRoutingService extends DiscoveryServiceListener { - - ServerAddress getCurrentServer(); - - Optional resolveById(EntityId entityId); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java deleted file mode 100644 index e114a9b6ab..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import com.google.common.hash.HashCode; -import com.google.common.hash.HashFunction; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.utils.MiscUtils; - -import javax.annotation.PostConstruct; -import java.util.Arrays; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -/** - * Cluster service implementation based on consistent hash ring - */ - -@Service -@Slf4j -public class ConsistentClusterRoutingService implements ClusterRoutingService { - - @Autowired - private DiscoveryService discoveryService; - - @Value("${cluster.hash_function_name}") - private String hashFunctionName; - @Value("${cluster.vitrual_nodes_size}") - private Integer virtualNodesSize; - - private ServerInstance currentServer; - - private HashFunction hashFunction; - - private ConsistentHashCircle[] circles; - private ConsistentHashCircle rootCircle; - - @PostConstruct - public void init() { - log.info("Initializing Cluster routing service!"); - this.hashFunction = MiscUtils.forName(hashFunctionName); - this.currentServer = discoveryService.getCurrentServer(); - this.circles = new ConsistentHashCircle[ServerType.values().length]; - for (ServerType serverType : ServerType.values()) { - circles[serverType.ordinal()] = new ConsistentHashCircle(); - } - rootCircle = circles[ServerType.CORE.ordinal()]; - addNode(discoveryService.getCurrentServer()); - for (ServerInstance instance : discoveryService.getOtherServers()) { - addNode(instance); - } - logCircle(); - log.info("Cluster routing service initialized!"); - } - - @Override - public ServerAddress getCurrentServer() { - return discoveryService.getCurrentServer().getServerAddress(); - } - - @Override - public Optional resolveById(EntityId entityId) { - return resolveByUuid(rootCircle, entityId.getId()); - } - - private Optional resolveByUuid(ConsistentHashCircle circle, UUID uuid) { - Assert.notNull(uuid); - if (circle.isEmpty()) { - return Optional.empty(); - } - Long hash = hashFunction.newHasher().putLong(uuid.getMostSignificantBits()) - .putLong(uuid.getLeastSignificantBits()).hash().asLong(); - if (!circle.containsKey(hash)) { - ConcurrentNavigableMap tailMap = - circle.tailMap(hash); - hash = tailMap.isEmpty() ? - circle.firstKey() : tailMap.firstKey(); - } - ServerInstance result = circle.get(hash); - if (!currentServer.equals(result)) { - return Optional.of(result.getServerAddress()); - } else { - return Optional.empty(); - } - } - - @Override - public void onServerAdded(ServerInstance server) { - log.info("On server added event: {}", server); - addNode(server); - logCircle(); - } - - @Override - public void onServerUpdated(ServerInstance server) { - log.debug("Ignoring server onUpdate event: {}", server); - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.info("On server removed event: {}", server); - removeNode(server); - logCircle(); - } - - private void addNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].put(hash(instance, i).asLong(), instance); - } - } - - private void removeNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].remove(hash(instance, i).asLong()); - } - } - - private HashCode hash(ServerInstance instance, int i) { - return hashFunction.newHasher().putString(instance.getHost(), MiscUtils.UTF8).putInt(instance.getPort()).putInt(i).hash(); - } - - private void logCircle() { - log.trace("Consistent Hash Circle Start"); - Arrays.asList(circles).forEach(ConsistentHashCircle::log); - log.trace("Consistent Hash Circle End"); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java deleted file mode 100644 index 23b050634c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import com.google.protobuf.ByteString; -import io.grpc.Server; -import io.grpc.ServerBuilder; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.service.cluster.discovery.ServerInstanceService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; - -import javax.annotation.PreDestroy; -import java.io.IOException; -import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -/** - * @author Andrew Shvayka - */ -@Service -@Slf4j -public class ClusterGrpcService extends ClusterRpcServiceGrpc.ClusterRpcServiceImplBase implements ClusterRpcService { - - @Autowired - private ServerInstanceService instanceService; - - @Autowired - private DataDecodingEncodingService encodingService; - - private RpcMsgListener listener; - - private Server server; - - private ServerInstance instance; - - private ConcurrentMap>> pendingSessionMap = - new ConcurrentHashMap<>(); - - public void init(RpcMsgListener listener) { - this.listener = listener; - log.info("Initializing RPC service!"); - instance = instanceService.getSelf(); - server = ServerBuilder.forPort(instance.getPort()).addService(this).build(); - log.info("Going to start RPC server using port: {}", instance.getPort()); - try { - server.start(); - } catch (IOException e) { - log.error("Failed to start RPC server!", e); - throw new RuntimeException("Failed to start RPC server!"); - } - log.info("RPC service initialized!"); - } - - @Override - public void onSessionCreated(UUID msgUid, StreamObserver inputStream) { - BlockingQueue> queue = pendingSessionMap.remove(msgUid); - if (queue != null) { - try { - queue.put(inputStream); - } catch (InterruptedException e) { - log.warn("Failed to report created session!"); - Thread.currentThread().interrupt(); - } - } else { - log.warn("Failed to lookup pending session!"); - } - } - - @Override - public StreamObserver handleMsgs( - StreamObserver responseObserver) { - log.info("Processing new session."); - return createSession(new RpcSessionCreateRequestMsg(UUID.randomUUID(), null, responseObserver)); - } - - - @PreDestroy - public void stop() { - if (server != null) { - log.info("Going to onStop RPC server"); - server.shutdownNow(); - try { - server.awaitTermination(); - log.info("RPC server stopped!"); - } catch (InterruptedException e) { - log.warn("Failed to onStop RPC server!"); - Thread.currentThread().interrupt(); - } - } - } - - - @Override - public void broadcast(RpcBroadcastMsg msg) { - listener.onBroadcastMsg(msg); - } - - private StreamObserver createSession(RpcSessionCreateRequestMsg msg) { - BlockingQueue> queue = new ArrayBlockingQueue<>(1); - pendingSessionMap.put(msg.getMsgUid(), queue); - listener.onRpcSessionCreateRequestMsg(msg); - try { - StreamObserver observer = queue.take(); - log.info("Processed new session."); - return observer; - } catch (Exception e) { - log.info("Failed to process session.", e); - throw new RuntimeException(e); - } - } - - @Override - public void tell(ClusterAPIProtos.ClusterMessage message) { - listener.onSendMsg(message); - } - - @Override - public void tell(ServerAddress serverAddress, TbActorMsg actorMsg) { - listener.onSendMsg(encodingService.convertToProtoDataMessage(serverAddress, actorMsg)); - } - - @Override - public void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data) { - ClusterAPIProtos.ClusterMessage msg = ClusterAPIProtos.ClusterMessage - .newBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(serverAddress.getHost()) - .setPort(serverAddress.getPort()) - .build()) - .setMessageType(msgType) - .setPayload(ByteString.copyFrom(data)).build(); - listener.onSendMsg(msg); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java deleted file mode 100644 index 637924fca9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import io.grpc.stub.StreamObserver; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -public interface ClusterRpcService { - - void init(RpcMsgListener listener); - - void broadcast(RpcBroadcastMsg msg); - - void onSessionCreated(UUID msgUid, StreamObserver inputStream); - - void tell(ClusterAPIProtos.ClusterMessage message); - - void tell(ServerAddress serverAddress, TbActorMsg actorMsg); - - void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java deleted file mode 100644 index aff0bc4a41..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import io.grpc.Channel; -import io.grpc.ManagedChannel; -import io.grpc.stub.StreamObserver; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.io.Closeable; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -@Slf4j -public final class GrpcSession implements Closeable { - private final UUID sessionId; - private final boolean client; - private final GrpcSessionListener listener; - private final ManagedChannel channel; - private StreamObserver inputStream; - private StreamObserver outputStream; - - private boolean connected; - private ServerAddress remoteServer; - - public GrpcSession(GrpcSessionListener listener) { - this(null, listener, null); - } - - public GrpcSession(ServerAddress remoteServer, GrpcSessionListener listener, ManagedChannel channel) { - this.sessionId = UUID.randomUUID(); - this.listener = listener; - if (remoteServer != null) { - this.client = true; - this.connected = true; - this.remoteServer = remoteServer; - } else { - this.client = false; - } - this.channel = channel; - } - - public void initInputStream() { - this.inputStream = new StreamObserver() { - @Override - public void onNext(ClusterAPIProtos.ClusterMessage clusterMessage) { - if (!connected && clusterMessage.getMessageType() == ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE) { - connected = true; - ServerAddress rpcAddress = new ServerAddress(clusterMessage.getServerAddress().getHost(), clusterMessage.getServerAddress().getPort(), ServerType.CORE); - remoteServer = new ServerAddress(rpcAddress.getHost(), rpcAddress.getPort(), ServerType.CORE); - listener.onConnected(GrpcSession.this); - } - if (connected) { - listener.onReceiveClusterGrpcMsg(GrpcSession.this, clusterMessage); - } - } - - @Override - public void onError(Throwable t) { - listener.onError(GrpcSession.this, t); - } - - @Override - public void onCompleted() { - outputStream.onCompleted(); - listener.onDisconnected(GrpcSession.this); - } - }; - } - - public void initOutputStream() { - if (client) { - listener.onConnected(GrpcSession.this); - } - } - - public void sendMsg(ClusterAPIProtos.ClusterMessage msg) { - if (connected) { - try { - outputStream.onNext(msg); - } catch (Throwable t) { - try { - outputStream.onError(t); - } catch (Throwable t2) { - } - listener.onError(GrpcSession.this, t); - } - } else { - log.warn("[{}] Failed to send message due to closed session!", sessionId); - } - } - - @Override - public void close() { - connected = false; - try { - outputStream.onCompleted(); - } catch (IllegalStateException e) { - log.debug("[{}] Failed to close output stream: {}", sessionId, e.getMessage()); - } - if (channel != null) { - channel.shutdownNow(); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java deleted file mode 100644 index a6ecf967d6..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -public interface GrpcSessionListener { - - void onConnected(GrpcSession session); - - void onDisconnected(GrpcSession session); - - void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage); - - void onError(GrpcSession session, Throwable t); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java deleted file mode 100644 index 2ff37b39e0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ - -public interface RpcMsgListener { - void onReceivedMsg(ServerAddress remoteServer, ClusterAPIProtos.ClusterMessage msg); - void onSendMsg(ClusterAPIProtos.ClusterMessage msg); - void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg); - void onBroadcastMsg(RpcBroadcastMsg msg); -} diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java index a5d3ab465e..4a781b8673 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java @@ -16,8 +16,6 @@ package org.thingsboard.server.service.encoding; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; @@ -27,8 +25,5 @@ public interface DataDecodingEncodingService { byte[] encode(TbActorMsg msq); - ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, - TbActorMsg msg); - } diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java index 45bb9f78fd..8d89059488 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java @@ -15,25 +15,19 @@ */ package org.thingsboard.server.service.encoding; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; - - @Slf4j @Service public class ProtoWithFSTService implements DataDecodingEncodingService { - private final FSTConfiguration config = FSTConfiguration.createDefaultConfiguration(); + @Override public Optional decode(byte[] byteArray) { try { @@ -42,7 +36,7 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { } catch (IllegalArgumentException e) { log.error("Error during deserialization message, [{}]", e.getMessage()); - return Optional.empty(); + return Optional.empty(); } } @@ -51,18 +45,4 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { return config.asByteArray(msq); } - @Override - public ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, - TbActorMsg msg) { - return ClusterAPIProtos.ClusterMessage - .newBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(serverAddress.getHost()) - .setPort(serverAddress.getPort()) - .build()) - .setMessageType(CLUSTER_ACTOR_MESSAGE) - .setPayload(ByteString.copyFrom(encode(msg))).build(); - - } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java index 721d43bf9f..573d152635 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java @@ -290,6 +290,20 @@ public class CassandraDatabaseUpgradeService extends AbstractCassandraDatabaseUp log.info("Attributes updated."); } catch (InvalidQueryException e) { } + + String updateTenantCoreTableStmt = "alter table tenant add isolated_tb_core boolean"; + String updateTenantRuleEngineTableStmt = "alter table tenant add isolated_tb_rule_engine boolean"; + + try { + log.info("Updating tenant..."); + cluster.getSession().execute(updateTenantCoreTableStmt); + Thread.sleep(2500); + + cluster.getSession().execute(updateTenantRuleEngineTableStmt); + Thread.sleep(2500); + log.info("Tenant updated."); + } catch (InvalidQueryException e) { + } log.info("Schema updated."); break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index ef03e3ec43..f673f498fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,6 +221,10 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } + try { + conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); + } catch (Exception e) { + } log.info("Schema updated."); } break; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java new file mode 100644 index 0000000000..926b08f38b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -0,0 +1,204 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Service +@Slf4j +public class DefaultTbClusterService implements TbClusterService { + + @Value("${cluster.stats.enabled:false}") + private boolean statsEnabled; + + private final AtomicInteger toCoreMsgs = new AtomicInteger(0); + private final AtomicInteger toCoreNfs = new AtomicInteger(0); + private final AtomicInteger toRuleEngineMsgs = new AtomicInteger(0); + private final AtomicInteger toRuleEngineNfs = new AtomicInteger(0); + private final AtomicInteger toTransportNfs = new AtomicInteger(0); + + private final TbQueueProducerProvider producerProvider; + private final PartitionService partitionService; + private final DataDecodingEncodingService encodingService; + + public DefaultTbClusterService(TbQueueProducerProvider producerProvider, PartitionService partitionService, DataDecodingEncodingService encodingService) { + this.producerProvider = producerProvider; + this.partitionService = partitionService; + this.encodingService = encodingService; + } + + @Override + public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + + @Override + public void pushMsgToCore(TopicPartitionInfo tpi, UUID msgId, ToCoreMsg msg, TbQueueCallback callback) { + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toCoreMsgs.incrementAndGet(); + } + + @Override + public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + log.trace("PUSHING msg: {} to:{}", msg, tpi); + byte[] msgBytes = encodingService.encode(msg); + ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback); + toCoreMsgs.incrementAndGet(); + } + + @Override + public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); + FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() + .setRequestIdMSB(response.getId().getMostSignificantBits()) + .setRequestIdLSB(response.getId().getLeastSignificantBits()) + .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); + response.getResponse().ifPresent(builder::setResponse); + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); + producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); + toCoreNfs.incrementAndGet(); + } + + @Override + public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) { + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toRuleEngineMsgs.incrementAndGet(); + } + + @Override + public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg tbMsg, TbQueueCallback callback) { + if (tenantId.isNullUid()) { + if (entityId.getEntityType().equals(EntityType.TENANT)) { + tenantId = new TenantId(entityId.getId()); + } else { + log.warn("[{}][{}] Received invalid message: {}", tenantId, entityId, tbMsg); + return; + } + } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + log.trace("PUSHING msg: {} to:{}", tbMsg, tpi); + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); + toRuleEngineMsgs.incrementAndGet(); + } + + @Override + public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); + FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() + .setRequestIdMSB(response.getId().getMostSignificantBits()) + .setRequestIdLSB(response.getId().getLeastSignificantBits()) + .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); + response.getResponse().ifPresent(builder::setResponse); + ToRuleEngineNotificationMsg msg = ToRuleEngineNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); + producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); + toRuleEngineNfs.incrementAndGet(); + } + + @Override + public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); + producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback); + toTransportNfs.incrementAndGet(); + } + + @Override + public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { + log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); + broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); + } + + private void broadcast(ComponentLifecycleMsg msg) { + byte[] msgBytes = encodingService.encode(msg); + TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); + Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); + if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)) { + TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); + Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); + for (String serviceId : tbCoreServices) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); + toCoreNfs.incrementAndGet(); + } + // No need to push notifications twice + tbRuleEngineServices.removeAll(tbCoreServices); + } + for (String serviceId : tbRuleEngineServices) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null); + toRuleEngineNfs.incrementAndGet(); + } + } + + @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + int toCoreMsgCnt = toCoreMsgs.getAndSet(0); + int toCoreNfsCnt = toCoreNfs.getAndSet(0); + int toRuleEngineMsgsCnt = toRuleEngineMsgs.getAndSet(0); + int toRuleEngineNfsCnt = toRuleEngineNfs.getAndSet(0); + int toTransportNfsCnt = toTransportNfs.getAndSet(0); + if (toCoreMsgCnt > 0 || toCoreNfsCnt > 0 || toRuleEngineMsgsCnt > 0 || toRuleEngineNfsCnt > 0 || toTransportNfsCnt > 0) { + log.info("To TbCore: [{}] messages [{}] notifications; To TbRuleEngine: [{}] messages [{}] notifications; To Transport: [{}] notifications", + toCoreMsgCnt, toCoreNfsCnt, toRuleEngineMsgsCnt, toRuleEngineNfsCnt, toTransportNfsCnt); + } + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java new file mode 100644 index 0000000000..c943e4c4d0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -0,0 +1,283 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import akka.actor.ActorRef; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; +import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; +import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.subscription.SubscriptionManagerService; +import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; +import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; + +import javax.annotation.PostConstruct; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@Slf4j +public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService { + + @Value("${queue.core.poll-interval}") + private long pollDuration; + @Value("${queue.core.pack-processing-timeout}") + private long packProcessingTimeout; + @Value("${queue.core.stats.enabled:false}") + private boolean statsEnabled; + + private final TbQueueConsumer> mainConsumer; + private final DeviceStateService stateService; + private final TbLocalSubscriptionService localSubscriptionService; + private final SubscriptionManagerService subscriptionManagerService; + private final TbCoreDeviceRpcService tbCoreDeviceRpcService; + private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); + + public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, + DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, + SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, + TbCoreDeviceRpcService tbCoreDeviceRpcService) { + super(actorContext, encodingService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); + this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); + this.stateService = stateService; + this.localSubscriptionService = localSubscriptionService; + this.subscriptionManagerService = subscriptionManagerService; + this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; + } + + @PostConstruct + public void init() { + super.init("tb-core-consumer", "tb-core-notifications-consumer"); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(getServiceType())) { + log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); + this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); + } + } + + @Override + protected void launchMainConsumers() { + consumersExecutor.submit(() -> { + while (!stopped) { + try { + List> msgs = mainConsumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + pendingMap.forEach((id, msg) -> { + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); + try { + ToCoreMsg toCoreMsg = msg.getValue(); + if (toCoreMsg.hasToSubscriptionMgrMsg()) { + log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); + forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); + } else if (toCoreMsg.hasToDeviceActorMsg()) { + log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); + forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); + } else if (toCoreMsg.hasDeviceStateServiceMsg()) { + log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); + forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); + } else if (toCoreMsg.getToDeviceActorNotificationMsg() != null && !toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + } + mainConsumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + } + log.info("TB Core Consumer stopped."); + }); + } + + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_CORE; + } + + @Override + protected long getNotificationPollDuration() { + return pollDuration; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { + ToCoreNotificationMsg toCoreNotification = msg.getValue(); + if (toCoreNotification.hasToLocalSubscriptionServiceMsg()) { + log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreNotification.getToLocalSubscriptionServiceMsg()); + forwardToLocalSubMgrService(toCoreNotification.getToLocalSubscriptionServiceMsg(), callback); + } else if (toCoreNotification.hasFromDeviceRpcResponse()) { + log.trace("[{}] Forwarding message to RPC service {}", id, toCoreNotification.getFromDeviceRpcResponse()); + forwardToCoreRpcService(toCoreNotification.getFromDeviceRpcResponse(), callback); + } else if (toCoreNotification.getComponentLifecycleMsg() != null && !toCoreNotification.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreNotification.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } + if (statsEnabled) { + stats.log(toCoreNotification); + } + } + + private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { + RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; + FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) + , proto.getResponse(), error); + tbCoreDeviceRpcService.processRpcResponseFromRuleEngine(response); + callback.onSuccess(); + } + + @Scheduled(fixedDelayString = "${queue.core.stats.print-interval-ms}") + public void printStats() { + if (statsEnabled) { + stats.printStats(); + } + } + + private void forwardToLocalSubMgrService(LocalSubscriptionServiceMsgProto msg, TbCallback callback) { + if (msg.hasSubUpdate()) { + localSubscriptionService.onSubscriptionUpdate(msg.getSubUpdate().getSessionId(), TbSubscriptionUtils.fromProto(msg.getSubUpdate()), callback); + } else { + throwNotHandled(msg, callback); + } + } + + private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbCallback callback) { + if (msg.hasAttributeSub()) { + subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getAttributeSub()), callback); + } else if (msg.hasTelemetrySub()) { + subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getTelemetrySub()), callback); + } else if (msg.hasSubClose()) { + TbSubscriptionCloseProto closeProto = msg.getSubClose(); + subscriptionManagerService.cancelSubscription(closeProto.getSessionId(), closeProto.getSubscriptionId(), callback); + } else if (msg.hasTsUpdate()) { + TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); + subscriptionManagerService.onTimeSeriesUpdate( + new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), + TbSubscriptionUtils.toTsKvEntityList(proto.getDataList()), callback); + } else if (msg.hasAttrUpdate()) { + TbAttributeUpdateProto proto = msg.getAttrUpdate(); + subscriptionManagerService.onAttributesUpdate( + new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), + proto.getScope(), TbSubscriptionUtils.toAttributeKvList(proto.getDataList()), callback); + } else { + throwNotHandled(msg, callback); + } + if (statsEnabled) { + stats.log(msg); + } + } + + private void forwardToStateService(DeviceStateServiceMsgProto deviceStateServiceMsg, TbCallback callback) { + if (statsEnabled) { + stats.log(deviceStateServiceMsg); + } + stateService.onQueueMsg(deviceStateServiceMsg, callback); + } + + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbCallback callback) { + if (statsEnabled) { + stats.log(toDeviceActorMsg); + } + actorContext.tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + } + + private void throwNotHandled(Object msg, TbCallback callback) { + log.warn("Message not handled: {}", msg); + callback.onFailure(new RuntimeException("Message not handled!")); + } + + @Override + protected void stopMainConsumers() { + if (mainConsumer != null) { + mainConsumer.unsubscribe(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java new file mode 100644 index 0000000000..6736518bcc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -0,0 +1,266 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import akka.actor.ActorRef; +import com.google.protobuf.ProtocolStringList; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceQueue; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.stats.RuleEngineStatisticsService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +@TbRuleEngineComponent +@Slf4j +public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { + + @Value("${queue.rule-engine.poll-interval}") + private long pollDuration; + @Value("${queue.rule-engine.pack-processing-timeout}") + private long packProcessingTimeout; + @Value("${queue.rule-engine.stats.enabled:true}") + private boolean statsEnabled; + + private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; + private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory; + private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final RuleEngineStatisticsService statisticsService; + private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); + private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); + private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); + private ExecutorService submitExecutor; + + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, + TbRuleEngineSubmitStrategyFactory submitStrategyFactory, + TbQueueRuleEngineSettings ruleEngineSettings, + TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, + ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { + super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); + this.statisticsService = statisticsService; + this.ruleEngineSettings = ruleEngineSettings; + this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; + this.submitStrategyFactory = submitStrategyFactory; + this.processingStrategyFactory = processingStrategyFactory; + } + + @PostConstruct + public void init() { + super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); + for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) { + consumerConfigurations.putIfAbsent(configuration.getName(), configuration); + consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); + consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName())); + } + submitExecutor = Executors.newSingleThreadExecutor(); + } + + @PreDestroy + public void stop() { + if (submitExecutor != null) { + submitExecutor.shutdownNow(); + } + + ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config)); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(getServiceType())) { + ServiceQueue serviceQueue = partitionChangeEvent.getServiceQueueKey().getServiceQueue(); + log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), partitionChangeEvent.getPartitions()); + consumers.get(serviceQueue.getQueue()).subscribe(partitionChangeEvent.getPartitions()); + } + } + + @Override + protected void launchMainConsumers() { + consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue))); + } + + @Override + protected void stopMainConsumers() { + consumers.values().forEach(TbQueueConsumer::unsubscribe); + } + + private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats) { + consumersExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = consumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + TbRuleEngineSubmitStrategy submitStrategy = submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy()); + TbRuleEngineProcessingStrategy ackStrategy = processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy()); + + submitStrategy.init(msgs); + + while (!stopped) { + ProcessingAttemptContext ctx = new ProcessingAttemptContext(submitStrategy); + submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> { + log.trace("[{}] Creating callback for message: {}", id, msg.getValue()); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); + TbMsgCallback callback = new TbMsgPackCallback(id, tenantId, ctx); + try { + if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { + forwardToRuleEngineActor(tenantId, toRuleEngineMsg, callback); + } else { + callback.onSuccess(); + } + } catch (Exception e) { + callback.onFailure(new RuleEngineException(e.getMessage())); + } + })); + + boolean timeout = false; + if (!ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { + timeout = true; + } + + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(timeout, ctx); + TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); + if (statsEnabled) { + stats.log(result, decision.isCommit()); + } + if (decision.isCommit()) { + submitStrategy.stop(); + break; + } else { + submitStrategy.update(decision.getReprocessMap()); + } + } + consumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to process messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + } + log.info("TB Rule Engine Consumer stopped."); + }); + } + + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_RULE_ENGINE; + } + + @Override + protected long getNotificationPollDuration() { + return pollDuration; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception { + ToRuleEngineNotificationMsg nfMsg = msg.getValue(); + if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(nfMsg.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } else { + callback.onSuccess(); + } + } + + private void forwardToRuleEngineActor(TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + TbMsg tbMsg = TbMsg.fromBytes(toRuleEngineMsg.getTbMsg().toByteArray(), callback); + QueueToRuleEngineMsg msg; + ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); + Set relationTypes = null; + if (relationTypesList != null) { + if (relationTypesList.size() == 1) { + relationTypes = Collections.singleton(relationTypesList.get(0)); + } else { + relationTypes = new HashSet<>(relationTypesList); + } + } + msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); + actorContext.tell(msg, ActorRef.noSender()); + } + + @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") + public void printStats() { + if (statsEnabled) { + long ts = System.currentTimeMillis(); + consumerStats.forEach((queue, stats) -> { + stats.printStats(); + statisticsService.reportQueueStats(ts, stats); + }); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java new file mode 100644 index 0000000000..aae3cef0ac --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'") +public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService { + + private final TenantService tenantService; + + public DefaultTenantRoutingInfoService(TenantService tenantService) { + this.tenantService = tenantService; + } + + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + Tenant tenant = tenantService.findTenantById(tenantId); + if (tenant != null) { + return new TenantRoutingInfo(tenantId, tenant.isIsolatedTbCore(), tenant.isIsolatedTbRuleEngine()); + } else { + throw new RuntimeException("Tenant not found!"); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java new file mode 100644 index 0000000000..aefb1697cd --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class ProcessingAttemptContext { + + private final TbRuleEngineSubmitStrategy submitStrategy; + + private final AtomicInteger pendingCount; + private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + @Getter + private final ConcurrentMap> pendingMap; + @Getter + private final ConcurrentMap> successMap = new ConcurrentHashMap<>(); + @Getter + private final ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + @Getter + private final ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); + + public ProcessingAttemptContext(TbRuleEngineSubmitStrategy submitStrategy) { + this.submitStrategy = submitStrategy; + this.pendingMap = submitStrategy.getPendingMap(); + this.pendingCount = new AtomicInteger(pendingMap.size()); + } + + public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { + return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); + } + + public void onSuccess(UUID id) { + TbProtoQueueMsg msg; + boolean empty = false; + msg = pendingMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + successMap.put(id, msg); + submitStrategy.onSuccess(id); + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } + + public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) { + TbProtoQueueMsg msg; + boolean empty = false; + msg = pendingMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + failedMap.put(id, msg); + exceptionsMap.putIfAbsent(tenantId, e); + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java new file mode 100644 index 0000000000..cc722720d5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; + +import java.util.UUID; + +public interface TbClusterService { + + void pushMsgToCore(TopicPartitionInfo tpi, UUID msgKey, ToCoreMsg msg, TbQueueCallback callback); + + void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback); + + void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback); + + void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); + + void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, TransportProtos.ToRuleEngineMsg msg, TbQueueCallback callback); + + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg msg, TbQueueCallback callback); + + void pushNotificationToRuleEngine(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); + + void pushNotificationToTransport(String targetServiceId, ToTransportMsg response, TbQueueCallback callback); + + void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java new file mode 100644 index 0000000000..2dcdb0e0c2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; + +public interface TbCoreConsumerService extends ApplicationListener { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java similarity index 63% rename from application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index dea6e6d33c..78319997a9 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; @@ -21,31 +21,26 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -public class RuleEngineStats { +public class TbCoreConsumerStats { private final AtomicInteger totalCounter = new AtomicInteger(0); private final AtomicInteger sessionEventCounter = new AtomicInteger(0); - private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); - private final AtomicInteger postAttributesCounter = new AtomicInteger(0); private final AtomicInteger getAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); - private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); + private final AtomicInteger deviceStateCounter = new AtomicInteger(0); + private final AtomicInteger subscriptionMsgCounter = new AtomicInteger(0); + private final AtomicInteger toCoreNotificationsCounter = new AtomicInteger(0); + public void log(TransportProtos.TransportToDeviceActorMsg msg) { totalCounter.incrementAndGet(); if (msg.hasSessionEvent()) { sessionEventCounter.incrementAndGet(); } - if (msg.hasPostTelemetry()) { - postTelemetryCounter.incrementAndGet(); - } - if (msg.hasPostAttributes()) { - postAttributesCounter.incrementAndGet(); - } if (msg.hasGetAttributes()) { getAttributesCounter.incrementAndGet(); } @@ -58,9 +53,6 @@ public class RuleEngineStats { if (msg.hasToDeviceRPCCallResponse()) { toDeviceRPCCallResponseCounter.incrementAndGet(); } - if (msg.hasToServerRPCCallRequest()) { - toServerRPCCallRequestCounter.incrementAndGet(); - } if (msg.hasSubscriptionInfo()) { subscriptionInfoCounter.incrementAndGet(); } @@ -69,15 +61,32 @@ public class RuleEngineStats { } } + public void log(TransportProtos.DeviceStateServiceMsgProto msg) { + totalCounter.incrementAndGet(); + deviceStateCounter.incrementAndGet(); + } + + public void log(TransportProtos.SubscriptionMgrMsgProto msg) { + totalCounter.incrementAndGet(); + subscriptionMsgCounter.incrementAndGet(); + } + + public void log(TransportProtos.ToCoreNotificationMsg msg) { + totalCounter.incrementAndGet(); + toCoreNotificationsCounter.incrementAndGet(); + } + public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { - log.info("Transport total [{}] sessionEvents [{}] telemetry [{}] attributes [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] " + - "toServerRpc [{}] subInfo [{}] claimDevice [{}] ", - total, sessionEventCounter.getAndSet(0), postTelemetryCounter.getAndSet(0), - postAttributesCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), + log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + + " deviceState [{}] subMgr [{}] coreNfs [{}]", + total, sessionEventCounter.getAndSet(0), + getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), - toServerRPCCallRequestCounter.getAndSet(0), subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); + subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0) + , deviceStateCounter.getAndSet(0), subscriptionMsgCounter.getAndSet(0), toCoreNotificationsCounter.getAndSet(0)); } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java new file mode 100644 index 0000000000..2a6b6a658d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.RuleNodeException; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; + +@Slf4j +public class TbMsgPackCallback implements TbMsgCallback { + private final UUID id; + private final TenantId tenantId; + private final ProcessingAttemptContext ctx; + + public TbMsgPackCallback(UUID id, TenantId tenantId, ProcessingAttemptContext ctx) { + this.id = id; + this.tenantId = tenantId; + this.ctx = ctx; + } + + @Override + public void onSuccess() { + log.trace("[{}] ON SUCCESS", id); + ctx.onSuccess(id); + } + + @Override + public void onFailure(RuleEngineException e) { + log.trace("[{}] ON FAILURE", id, e); + ctx.onFailure(tenantId, id, e); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java new file mode 100644 index 0000000000..ba5a883ea0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TbCallback; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; + +@Slf4j +public class TbPackCallback implements TbCallback { + private final CountDownLatch processingTimeoutLatch; + private final ConcurrentMap ackMap; + private final ConcurrentMap failedMap; + private final UUID id; + + public TbPackCallback(UUID id, + CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap failedMap) { + this.id = id; + this.processingTimeoutLatch = processingTimeoutLatch; + this.ackMap = ackMap; + this.failedMap = failedMap; + } + + @Override + public void onSuccess() { + log.trace("[{}] ON SUCCESS", id); + T msg = ackMap.remove(id); + if (msg != null && ackMap.isEmpty()) { + processingTimeoutLatch.countDown(); + } + } + + @Override + public void onFailure(Throwable t) { + log.trace("[{}] ON FAILURE", id, t); + T msg = ackMap.remove(id); + if (msg != null) { + failedMap.put(id, msg); + } + if (ackMap.isEmpty()) { + processingTimeoutLatch.countDown(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java new file mode 100644 index 0000000000..19966b4566 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; + +public interface TbRuleEngineConsumerService extends ApplicationListener { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java new file mode 100644 index 0000000000..40017d2b40 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -0,0 +1,132 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +@Data +public class TbRuleEngineConsumerStats { + + public static final String TOTAL_MSGS = "totalMsgs"; + public static final String SUCCESSFUL_MSGS = "successfulMsgs"; + public static final String TMP_TIMEOUT = "tmpTimeout"; + public static final String TMP_FAILED = "tmpFailed"; + public static final String TIMEOUT_MSGS = "timeoutMsgs"; + public static final String FAILED_MSGS = "failedMsgs"; + public static final String SUCCESSFUL_ITERATIONS = "successfulIterations"; + public static final String FAILED_ITERATIONS = "failedIterations"; + + private final AtomicInteger totalMsgCounter = new AtomicInteger(0); + private final AtomicInteger successMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger failedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger successIterationsCounter = new AtomicInteger(0); + private final AtomicInteger failedIterationsCounter = new AtomicInteger(0); + + private final Map counters = new HashMap<>(); + private final ConcurrentMap tenantStats = new ConcurrentHashMap<>(); + private final ConcurrentMap tenantExceptions = new ConcurrentHashMap<>(); + + private final String queueName; + + public TbRuleEngineConsumerStats(String queueName) { + this.queueName = queueName; + counters.put(TOTAL_MSGS, totalMsgCounter); + counters.put(SUCCESSFUL_MSGS, successMsgCounter); + counters.put(TIMEOUT_MSGS, timeoutMsgCounter); + counters.put(FAILED_MSGS, failedMsgCounter); + + counters.put(TMP_TIMEOUT, tmpTimeoutMsgCounter); + counters.put(TMP_FAILED, tmpFailedMsgCounter); + counters.put(SUCCESSFUL_ITERATIONS, successIterationsCounter); + counters.put(FAILED_ITERATIONS, failedIterationsCounter); + } + + public void log(TbRuleEngineProcessingResult msg, boolean finalIterationForPack) { + int success = msg.getSuccessMap().size(); + int pending = msg.getPendingMap().size(); + int failed = msg.getFailedMap().size(); + totalMsgCounter.addAndGet(success + pending + failed); + successMsgCounter.addAndGet(success); + msg.getSuccessMap().values().forEach(m -> getTenantStats(m).logSuccess()); + if (finalIterationForPack) { + if (pending > 0 || failed > 0) { + timeoutMsgCounter.addAndGet(pending); + failedMsgCounter.addAndGet(failed); + if (pending > 0) { + msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTimeout()); + } + if (failed > 0) { + msg.getFailedMap().values().forEach(m -> getTenantStats(m).logFailed()); + } + failedIterationsCounter.incrementAndGet(); + } else { + successIterationsCounter.incrementAndGet(); + } + } else { + failedIterationsCounter.incrementAndGet(); + tmpTimeoutMsgCounter.addAndGet(pending); + tmpFailedMsgCounter.addAndGet(failed); + if (pending > 0) { + msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTmpTimeout()); + } + if (failed > 0) { + msg.getFailedMap().values().forEach(m -> getTenantStats(m).logTmpFailed()); + } + } + msg.getExceptionsMap().forEach(tenantExceptions::putIfAbsent); + } + + private TbTenantRuleEngineStats getTenantStats(TbProtoQueueMsg m) { + ToRuleEngineMsg reMsg = m.getValue(); + return tenantStats.computeIfAbsent(new UUID(reMsg.getTenantIdMSB(), reMsg.getTenantIdLSB()), TbTenantRuleEngineStats::new); + } + + public void printStats() { + int total = totalMsgCounter.get(); + if (total > 0) { + StringBuilder stats = new StringBuilder(); + counters.forEach((label, value) -> { + stats.append(label).append(" = [").append(value.get()).append("] "); + }); + log.info("[{}] Stats: {}", queueName, stats); + } + } + + public void reset() { + counters.values().forEach(counter -> counter.set(0)); + tenantStats.clear(); + tenantExceptions.clear(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java new file mode 100644 index 0000000000..fb36f5064b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +@Data +public class TbTenantRuleEngineStats { + + private final UUID tenantId; + + private final AtomicInteger totalMsgCounter = new AtomicInteger(0); + private final AtomicInteger successMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger failedMsgCounter = new AtomicInteger(0); + + private final Map counters = new HashMap<>(); + + public TbTenantRuleEngineStats(UUID tenantId) { + this.tenantId = tenantId; + counters.put(TbRuleEngineConsumerStats.TOTAL_MSGS, totalMsgCounter); + counters.put(TbRuleEngineConsumerStats.SUCCESSFUL_MSGS, successMsgCounter); + counters.put(TbRuleEngineConsumerStats.TIMEOUT_MSGS, timeoutMsgCounter); + counters.put(TbRuleEngineConsumerStats.FAILED_MSGS, failedMsgCounter); + + counters.put(TbRuleEngineConsumerStats.TMP_TIMEOUT, tmpTimeoutMsgCounter); + counters.put(TbRuleEngineConsumerStats.TMP_FAILED, tmpFailedMsgCounter); + } + + public void logSuccess() { + totalMsgCounter.incrementAndGet(); + successMsgCounter.incrementAndGet(); + } + + public void logFailed() { + totalMsgCounter.incrementAndGet(); + failedMsgCounter.incrementAndGet(); + } + + public void logTimeout() { + totalMsgCounter.incrementAndGet(); + timeoutMsgCounter.incrementAndGet(); + } + + public void logTmpFailed() { + totalMsgCounter.incrementAndGet(); + tmpFailedMsgCounter.incrementAndGet(); + } + + public void logTmpTimeout() { + totalMsgCounter.incrementAndGet(); + tmpTimeoutMsgCounter.incrementAndGet(); + } + + public void printStats() { + int total = totalMsgCounter.get(); + if (total > 0) { + StringBuilder stats = new StringBuilder(); + counters.forEach((label, value) -> { + stats.append(label).append(" = [").append(value.get()).append("]"); + }); + log.info("[{}] Stats: {}", tenantId, stats); + } + } + + public void reset() { + counters.values().forEach(counter -> counter.set(0)); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java new file mode 100644 index 0000000000..a238e34337 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.EventListener; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.TbPackCallback; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public abstract class AbstractConsumerService implements ApplicationListener { + + protected volatile ExecutorService consumersExecutor; + protected volatile ExecutorService notificationsConsumerExecutor; + protected volatile boolean stopped = false; + + protected final ActorSystemContext actorContext; + protected final DataDecodingEncodingService encodingService; + + protected final TbQueueConsumer> nfConsumer; + + public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbQueueConsumer> nfConsumer) { + this.actorContext = actorContext; + this.encodingService = encodingService; + this.nfConsumer = nfConsumer; + } + + public void init(String mainConsumerThreadName, String nfConsumerThreadName) { + this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName(mainConsumerThreadName)); + this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(nfConsumerThreadName)); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); + this.nfConsumer.subscribe(); + launchNotificationsConsumer(); + launchMainConsumers(); + } + + protected abstract ServiceType getServiceType(); + + protected abstract void launchMainConsumers(); + + protected abstract void stopMainConsumers(); + + protected abstract long getNotificationPollDuration(); + + protected abstract long getNotificationPackProcessingTimeout(); + + protected void launchNotificationsConsumer() { + notificationsConsumerExecutor.submit(() -> { + while (!stopped) { + try { + List> msgs = nfConsumer.poll(getNotificationPollDuration()); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + pendingMap.forEach((id, msg) -> { + log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); + try { + handleNotification(id, msg, callback); + } catch (Throwable e) { + log.warn("[{}] Failed to process notification: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { + pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); + failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); + } + nfConsumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain notifications from queue.", e); + try { + Thread.sleep(getNotificationPollDuration()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new notifications", e2); + } + } + } + } + log.info("TB Notifications Consumer stopped."); + }); + } + + protected abstract void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception; + + @PreDestroy + public void destroy() { + stopped = true; + + stopMainConsumers(); + + if (nfConsumer != null) { + nfConsumer.unsubscribe(); + } + + if (consumersExecutor != null) { + consumersExecutor.shutdownNow(); + } + if (notificationsConsumerExecutor != null) { + notificationsConsumerExecutor.shutdownNow(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..bef733ec22 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngineSubmitStrategy { + + protected final String queueName; + protected List orderedMsgList; + private volatile boolean stopped; + + public AbstractTbRuleEngineSubmitStrategy(String queueName) { + this.queueName = queueName; + } + + protected abstract void doOnSuccess(UUID id); + + @Override + public void init(List> msgs) { + orderedMsgList = msgs.stream().map(msg -> new IdMsgPair(UUID.randomUUID(), msg)).collect(Collectors.toList()); + } + + @Override + public ConcurrentMap> getPendingMap() { + return orderedMsgList.stream().collect(Collectors.toConcurrentMap(pair -> pair.uuid, pair -> pair.msg)); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + List newOrderedMsgList = new ArrayList<>(reprocessMap.size()); + for (IdMsgPair pair : orderedMsgList) { + if (reprocessMap.containsKey(pair.uuid)) { + newOrderedMsgList.add(pair); + } + } + orderedMsgList = newOrderedMsgList; + } + + @Override + public void onSuccess(UUID id) { + if (!stopped) { + doOnSuccess(id); + } + } + + @Override + public void stop() { + stopped = true; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..d0b1f7f99a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +@Slf4j +public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private final int batchSize; + private final AtomicInteger packIdx = new AtomicInteger(0); + private final Map> pendingPack = new LinkedHashMap<>(); + private volatile BiConsumer> msgConsumer; + + public BatchTbRuleEngineSubmitStrategy(String queueName, int batchSize) { + super(queueName); + this.batchSize = batchSize; + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + submitNext(); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + packIdx.set(0); + } + + @Override + protected void doOnSuccess(UUID id) { + boolean endOfPendingPack; + synchronized (pendingPack) { + TbProtoQueueMsg msg = pendingPack.remove(id); + endOfPendingPack = msg != null && pendingPack.isEmpty(); + } + if (endOfPendingPack) { + packIdx.incrementAndGet(); + submitNext(); + } + } + + private void submitNext() { + int listSize = orderedMsgList.size(); + int startIdx = Math.min(packIdx.get() * batchSize, listSize); + int endIdx = Math.min(startIdx + batchSize, listSize); + synchronized (pendingPack) { + pendingPack.clear(); + for (int i = startIdx; i < endIdx; i++) { + IdMsgPair pair = orderedMsgList.get(i); + pendingPack.put(pair.uuid, pair.msg); + } + } + int submitSize = pendingPack.size(); + if (log.isInfoEnabled() && submitSize > 0) { + log.info("[{}] submitting [{}] messages to rule engine", queueName, submitSize); + } + pendingPack.forEach(msgConsumer); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ffd1dd49d1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + public BurstTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + if (log.isInfoEnabled()) { + log.info("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); + } + orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg)); + } + + @Override + protected void doOnSuccess(UUID id) { + + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java similarity index 64% rename from application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java rename to application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java index 9f08463f91..2b2c203ec5 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.service.queue.processing; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.io.IOException; +import java.util.UUID; -/** - * Created by ashvayka on 05.10.18. - */ -public class ToRuleEngineMsgDecoder implements TbKafkaDecoder { - @Override - public ToRuleEngineMsg decode(byte[] data) throws IOException { - return ToRuleEngineMsg.parseFrom(data); +public class IdMsgPair { + final UUID uuid; + final TbProtoQueueMsg msg; + + public IdMsgPair(UUID uuid, TbProtoQueueMsg msg) { + this.uuid = uuid; + this.msg = msg; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ae5993cb1c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,108 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +@Slf4j +public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private volatile BiConsumer> msgConsumer; + private volatile ConcurrentMap msgToEntityIdMap = new ConcurrentHashMap<>(); + private volatile ConcurrentMap> entityIdToListMap = new ConcurrentHashMap<>(); + + public SequentialByEntityIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void init(List> msgs) { + super.init(msgs); + initMaps(); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + entityIdToListMap.forEach((entityId, queue) -> { + IdMsgPair msg = queue.peek(); + if (msg != null) { + msgConsumer.accept(msg.uuid, msg.msg); + } + }); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + initMaps(); + } + + @Override + protected void doOnSuccess(UUID id) { + EntityId entityId = msgToEntityIdMap.get(id); + if (entityId != null) { + Queue queue = entityIdToListMap.get(entityId); + if (queue != null) { + IdMsgPair next = null; + synchronized (queue) { + IdMsgPair expected = queue.peek(); + if (expected != null && expected.uuid.equals(id)) { + queue.poll(); + next = queue.peek(); + } + } + if (next != null) { + msgConsumer.accept(next.uuid, next.msg); + } + } + } + } + + private void initMaps() { + msgToEntityIdMap.clear(); + entityIdToListMap.clear(); + for (IdMsgPair pair : orderedMsgList) { + EntityId entityId = getEntityId(pair.msg.getValue()); + if (entityId != null) { + msgToEntityIdMap.put(pair.uuid, entityId); + entityIdToListMap.computeIfAbsent(entityId, id -> new LinkedList<>()).add(pair); + } + } + } + + protected abstract EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..cd8a97e82c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.UUID; + +@Slf4j +public class SequentialByOriginatorIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { + + public SequentialByOriginatorIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { + try { + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg()); + return EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + } catch (InvalidProtocolBufferException e) { + log.warn("[{}] Failed to parse TbMsg: {}", queueName, msg); + return null; + } + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java similarity index 55% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java rename to application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java index 3326c33aa3..b258c6db1b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java @@ -13,28 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.service.queue.processing; -import lombok.Data; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos; -@Data -public class SendToClusterMsg implements TbActorMsg { +import java.util.UUID; - private TbActorMsg msg; - private EntityId entityId; +public class SequentialByTenantIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { - public SendToClusterMsg(EntityId entityId, TbActorMsg msg) { - this.entityId = entityId; - this.msg = msg; + public SequentialByTenantIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); } - @Override - public MsgType getMsgType() { - return MsgType.SEND_TO_CLUSTER_MSG; + protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { + return new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ef45b983fc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +@Slf4j +public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private final AtomicInteger msgIdx = new AtomicInteger(0); + private volatile BiConsumer> msgConsumer; + private volatile UUID expectedMsgId; + + public SequentialTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + msgIdx.set(0); + submitNext(); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + } + + @Override + protected void doOnSuccess(UUID id) { + if (expectedMsgId.equals(id)) { + msgIdx.incrementAndGet(); + submitNext(); + } + } + + private void submitNext() { + int listSize = orderedMsgList.size(); + int idx = msgIdx.get(); + if (idx < listSize) { + IdMsgPair pair = orderedMsgList.get(idx); + expectedMsgId = pair.uuid; + if (log.isInfoEnabled()) { + log.info("[{}] submitting [{}] message to rule engine", queueName, pair.msg); + } + msgConsumer.accept(pair.uuid, pair.msg); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java new file mode 100644 index 0000000000..4a4829f0c2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.Data; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; + +@Data +public class TbRuleEngineProcessingDecision { + + private final boolean commit; + private final ConcurrentMap> reprocessMap; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java new file mode 100644 index 0000000000..8e0fcaa74a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.ProcessingAttemptContext; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; + +public class TbRuleEngineProcessingResult { + + @Getter + private final boolean success; + @Getter + private final boolean timeout; + @Getter + private final ProcessingAttemptContext ctx; + + public TbRuleEngineProcessingResult(boolean timeout, ProcessingAttemptContext ctx) { + this.timeout = timeout; + this.ctx = ctx; + this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty(); + } + + public ConcurrentMap> getPendingMap() { + return ctx.getPendingMap(); + } + + public ConcurrentMap> getSuccessMap() { + return ctx.getSuccessMap(); + } + + public ConcurrentMap> getFailedMap() { + return ctx.getFailedMap(); + } + + public ConcurrentMap getExceptionsMap() { + return ctx.getExceptionsMap(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java new file mode 100644 index 0000000000..cfba85f5dc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +public interface TbRuleEngineProcessingStrategy { + + TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java new file mode 100644 index 0000000000..bbf283e962 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -0,0 +1,140 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +@Component +@Slf4j +public class TbRuleEngineProcessingStrategyFactory { + + public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) { + switch (configuration.getType()) { + case "SKIP_ALL_FAILURES": + return new SkipStrategy(name); + case "RETRY_ALL": + return new RetryStrategy(name, true, true, true, configuration); + case "RETRY_FAILED": + return new RetryStrategy(name, false, true, false, configuration); + case "RETRY_TIMED_OUT": + return new RetryStrategy(name, false, false, true, configuration); + case "RETRY_FAILED_AND_TIMED_OUT": + return new RetryStrategy(name, false, true, true, configuration); + default: + throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); + } + } + + private static class RetryStrategy implements TbRuleEngineProcessingStrategy { + private final String queueName; + private final boolean retrySuccessful; + private final boolean retryFailed; + private final boolean retryTimeout; + private final int maxRetries; + private final double maxAllowedFailurePercentage; + private final long pauseBetweenRetries; + + private int initialTotalCount; + private int retryCount; + + public RetryStrategy(String queueName, boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, TbRuleEngineQueueAckStrategyConfiguration configuration) { + this.queueName = queueName; + this.retrySuccessful = retrySuccessful; + this.retryFailed = retryFailed; + this.retryTimeout = retryTimeout; + this.maxRetries = configuration.getRetries(); + this.maxAllowedFailurePercentage = configuration.getFailurePercentage(); + this.pauseBetweenRetries = configuration.getPauseBetweenRetries(); + } + + @Override + public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { + if (result.isSuccess()) { + return new TbRuleEngineProcessingDecision(true, null); + } else { + if (retryCount == 0) { + initialTotalCount = result.getPendingMap().size() + result.getFailedMap().size() + result.getSuccessMap().size(); + } + retryCount++; + double failedCount = result.getFailedMap().size() + result.getPendingMap().size(); + if (maxRetries > 0 && retryCount > maxRetries) { + log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); + return new TbRuleEngineProcessingDecision(true, null); + } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { + log.info("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); + return new TbRuleEngineProcessingDecision(true, null); + } else { + ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); + if (retryFailed) { + result.getFailedMap().forEach(toReprocess::put); + } + if (retryTimeout) { + result.getPendingMap().forEach(toReprocess::put); + } + if (retrySuccessful) { + result.getSuccessMap().forEach(toReprocess::put); + } + log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); + if (log.isTraceEnabled()) { + toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + if (pauseBetweenRetries > 0) { + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(pauseBetweenRetries)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return new TbRuleEngineProcessingDecision(false, toReprocess); + } + } + } + } + + private static class SkipStrategy implements TbRuleEngineProcessingStrategy { + + private final String queueName; + + public SkipStrategy(String name) { + this.queueName = name; + } + + @Override + public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { + if (!result.isSuccess()) { + log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + } + if (log.isTraceEnabled()) { + result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + if (log.isTraceEnabled()) { + result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + return new TbRuleEngineProcessingDecision(true, null); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..7b22da97db --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; + +public interface TbRuleEngineSubmitStrategy { + + void init(List> msgs); + + ConcurrentMap> getPendingMap(); + + void submitAttempt(BiConsumer> msgConsumer); + + void update(ConcurrentMap> reprocessMap); + + void onSuccess(UUID id); + + void stop(); +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java new file mode 100644 index 0000000000..f5a7457c17 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueSubmitStrategyConfiguration; + +@Component +@Slf4j +public class TbRuleEngineSubmitStrategyFactory { + + public TbRuleEngineSubmitStrategy newInstance(String name, TbRuleEngineQueueSubmitStrategyConfiguration configuration) { + switch (configuration.getType()) { + case "BURST": + return new BurstTbRuleEngineSubmitStrategy(name); + case "BATCH": + return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize()); + case "SEQUENTIAL_WITHIN_ORIGINATOR": + return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name); + case "SEQUENTIAL_WITHIN_TENANT": + return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name); + case "SEQUENTIAL": + return new SequentialTbRuleEngineSubmitStrategy(name); + default: + throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java deleted file mode 100644 index 5d303b54ab..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java +++ /dev/null @@ -1,222 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; -import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 27.03.18. - */ -@Service -@Slf4j -public class DefaultDeviceRpcService implements DeviceRpcService { - - private static final ObjectMapper json = new ObjectMapper(); - - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService rpcService; - - @Autowired - private DeviceService deviceService; - - @Autowired - @Lazy - private ActorService actorService; - - private ScheduledExecutorService rpcCallBackExecutor; - - private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); - private final ConcurrentMap> localToDeviceRpcRequests = new ConcurrentHashMap<>(); - - @PostConstruct - public void initExecutor() { - rpcCallBackExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rpc-callback")); - } - - @PreDestroy - public void shutdownExecutor() { - if (rpcCallBackExecutor != null) { - rpcCallBackExecutor.shutdownNow(); - } - } - - @Override - public void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToRuleEngineRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToRuleEngine(request); - scheduleTimeout(request, requestId, localToRuleEngineRpcRequests); - } - - @Override - public void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), requestOriginAddress); - if (routingService.getCurrentServer().equals(requestOriginAddress)) { - UUID requestId = response.getId(); - Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } else { - ClusterAPIProtos.FromDeviceRPCResponseProto.Builder builder = ClusterAPIProtos.FromDeviceRPCResponseProto.newBuilder(); - builder.setRequestIdMSB(response.getId().getMostSignificantBits()); - builder.setRequestIdLSB(response.getId().getLeastSignificantBits()); - response.getResponse().ifPresent(builder::setResponse); - if (response.getError().isPresent()) { - builder.setError(response.getError().get().ordinal()); - } else { - builder.setError(-1); - } - rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); - } - } - - @Override - public void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToDeviceRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToDevice(request); - scheduleTimeout(request, requestId, localToDeviceRpcRequests); - } - - @Override - public void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); - UUID requestId = response.getId(); - Consumer consumer = localToDeviceRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } - - @Override - public void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.FromDeviceRPCResponseProto proto; - try { - proto = ClusterAPIProtos.FromDeviceRPCResponseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; - FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), proto.getResponse(), error); - processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); - } - - @Override - public void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body) { - ToServerRpcResponseActorMsg rpcMsg = new ToServerRpcResponseActorMsg(tenantId, deviceId, new ToServerRpcResponseMsg(requestId, body)); - forward(deviceId, rpcMsg); - } - - private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { - ObjectNode entityNode = json.createObjectNode(); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("requestUUID", msg.getId().toString()); - metaData.putValue("originHost", routingService.getCurrentServer().getHost()); - metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); - metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); - metaData.putValue("oneway", Boolean.toString(msg.isOneway())); - - Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); - if (device != null) { - metaData.putValue("deviceName", device.getName()); - metaData.putValue("deviceType", device.getType()); - } - - entityNode.put("method", msg.getBody().getMethod()); - entityNode.put("params", msg.getBody().getParams()); - - try { - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON - , json.writeValueAsString(entityNode) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new ServiceToRuleEngineMsg(msg.getTenantId(), tbMsg))); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { - ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); - log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); - forward(msg.getDeviceId(), rpcMsg); - } - - private void forward(DeviceId deviceId, T msg) { - actorService.onMsg(new SendToClusterMsg(deviceId, msg)); - } - - private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId, ConcurrentMap> requestsMap) { - long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); - rpcCallBackExecutor.schedule(() -> { - log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); - Consumer consumer = requestsMap.remove(requestId); - if (consumer != null) { - consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); - } - }, timeout, TimeUnit.MILLISECONDS); - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java new file mode 100644 index 0000000000..f6e5eb8b43 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -0,0 +1,199 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import akka.actor.ActorRef; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 27.03.18. + */ +@Service +@Slf4j +@TbCoreComponent +public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { + + private static final ObjectMapper json = new ObjectMapper(); + + private final DeviceService deviceService; + private final TbClusterService clusterService; + private final TbServiceInfoProvider serviceInfoProvider; + private final ActorSystemContext actorContext; + + private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); + private final ConcurrentMap localToDeviceRpcRequests = new ConcurrentHashMap<>(); + + private Optional tbRuleEngineRpcService; + private ScheduledExecutorService scheduler; + private String serviceId; + + public DefaultTbCoreDeviceRpcService(DeviceService deviceService, TbClusterService clusterService, TbServiceInfoProvider serviceInfoProvider, + ActorSystemContext actorContext) { + this.deviceService = deviceService; + this.clusterService = clusterService; + this.serviceInfoProvider = serviceInfoProvider; + this.actorContext = actorContext; + } + + @Autowired(required = false) + public void setTbRuleEngineRpcService(Optional tbRuleEngineRpcService) { + this.tbRuleEngineRpcService = tbRuleEngineRpcService; + } + + @PostConstruct + public void initExecutor() { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-core-rpc-scheduler")); + serviceId = serviceInfoProvider.getServiceId(); + } + + @PreDestroy + public void shutdownExecutor() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToRuleEngineRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToRuleEngine(request); + scheduleToRuleEngineTimeout(request, requestId); + } + + @Override + public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), response); + UUID requestId = response.getId(); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + @Override + public void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg rpcMsg) { + ToDeviceRpcRequest request = rpcMsg.getMsg(); + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToDeviceRpcRequests.put(requestId, rpcMsg); + actorContext.tell(rpcMsg, ActorRef.noSender()); + scheduleToDeviceTimeout(request, requestId); + } + + @Override + public void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); + UUID requestId = response.getId(); + ToDeviceRpcRequestActorMsg request = localToDeviceRpcRequests.remove(requestId); + if (request != null) { + sendRpcResponseToTbRuleEngine(request.getServiceId(), response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + private void sendRpcResponseToTbRuleEngine(String originServiceId, FromDeviceRpcResponse response) { + if (serviceId.equals(originServiceId)) { + if (tbRuleEngineRpcService.isPresent()) { + tbRuleEngineRpcService.get().processRpcResponseFromDevice(response); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + clusterService.pushNotificationToRuleEngine(originServiceId, response, null); + } + } + + private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { + ObjectNode entityNode = json.createObjectNode(); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("requestUUID", msg.getId().toString()); + metaData.putValue("originServiceId", serviceId); + metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); + metaData.putValue("oneway", Boolean.toString(msg.isOneway())); + + Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); + if (device != null) { + metaData.putValue("deviceName", device.getName()); + metaData.putValue("deviceType", device.getType()); + } + + entityNode.put("method", msg.getBody().getMethod()); + entityNode.put("params", msg.getBody().getParams()); + + try { + TbMsg tbMsg = TbMsg.newMsg(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); + clusterService.pushMsgToRuleEngine(msg.getTenantId(), msg.getDeviceId(), tbMsg, null); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing to rule engine request.", requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout for processing to rule engine request.", requestId); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); + } + }, timeout, TimeUnit.MILLISECONDS); + } + + private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing to device request.", requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout for to device request.", requestId); + localToDeviceRpcRequests.remove(requestId); + }, timeout, TimeUnit.MILLISECONDS); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java new file mode 100644 index 0000000000..0ec730b7dc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -0,0 +1,177 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; +import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.queue.TbClusterService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +@Service +@TbRuleEngineComponent +@Slf4j +public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcService { + + private final PartitionService partitionService; + private final TbClusterService clusterService; + private final TbServiceInfoProvider serviceInfoProvider; + + + private final ConcurrentMap> toDeviceRpcRequests = new ConcurrentHashMap<>(); + + private Optional tbCoreRpcService; + private ScheduledExecutorService scheduler; + private String serviceId; + + public DefaultTbRuleEngineRpcService(PartitionService partitionService, + TbClusterService clusterService, + TbServiceInfoProvider serviceInfoProvider) { + this.partitionService = partitionService; + this.clusterService = clusterService; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Autowired(required = false) + public void setTbCoreRpcService(Optional tbCoreRpcService) { + this.tbCoreRpcService = tbCoreRpcService; + } + + @PostConstruct + public void initExecutor() { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rule-engine-rpc-scheduler")); + serviceId = serviceInfoProvider.getServiceId(); + } + + @PreDestroy + public void shutdownExecutor() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body) { + TransportProtos.ToServerRpcResponseMsg responseMsg = TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(requestId) + .setPayload(body).build(); + TransportProtos.ToTransportMsg msg = TransportProtos.ToTransportMsg.newBuilder() + .setSessionIdMSB(sessionId.getMostSignificantBits()) + .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setToServerResponse(responseMsg) + .build(); + clusterService.pushNotificationToTransport(serviceId, msg, null); + } + + @Override + public void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest src, Consumer consumer) { + ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), src.getTenantId(), src.getDeviceId(), + src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); + forwardRpcRequestToDeviceActor(request, response -> { + if (src.isRestApiCall()) { + sendRpcResponseToTbCore(src.getOriginServiceId(), response); + } + consumer.accept(RuleEngineDeviceRpcResponse.builder() + .deviceId(src.getDeviceId()) + .requestId(src.getRequestId()) + .error(response.getError()) + .response(response.getResponse()) + .build()); + }); + } + + @Override + public void processRpcResponseFromDevice(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from Core RPC Service", response.getId()); + UUID requestId = response.getId(); + Consumer consumer = toDeviceRpcRequests.remove(requestId); + if (consumer != null) { + scheduler.submit(() -> consumer.accept(response)); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + private void forwardRpcRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + toDeviceRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToDevice(request); + scheduleTimeout(request, requestId); + } + + private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(serviceId, msg); + if (tpi.isMyPartition()) { + log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); + if (tbCoreRpcService.isPresent()) { + tbCoreRpcService.get().forwardRpcRequestToDeviceActor(rpcMsg); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + log.trace("[{}] Forwarding msg {} to queue actor!", msg.getDeviceId(), msg); + clusterService.pushMsgToCore(rpcMsg, null); + } + } + + private void sendRpcResponseToTbCore(String originServiceId, FromDeviceRpcResponse response) { + if (serviceId.equals(originServiceId)) { + if (tbCoreRpcService.isPresent()) { + tbCoreRpcService.get().processRpcResponseFromRuleEngine(response); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + clusterService.pushNotificationToCore(originServiceId, response, null); + } + } + + private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); + Consumer consumer = toDeviceRpcRequests.remove(requestId); + if (consumer != null) { + scheduler.submit(() -> consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT))); + } + }, timeout, TimeUnit.MILLISECONDS); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java deleted file mode 100644 index feb5d58c3d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; - -import java.util.function.Consumer; - -/** - * Created by ashvayka on 16.04.18. - */ -public interface DeviceRpcService { - - void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer); - - void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response); - - void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer); - - void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response); - - void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data); - - void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body); -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java index 78ea0c5660..c1e5e2e038 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java @@ -19,7 +19,6 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import java.util.Optional; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java new file mode 100644 index 0000000000..896745c48e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; + +import java.util.function.Consumer; + +/** + * Handles REST API calls that contain RPC requests to Device. + */ +public interface TbCoreDeviceRpcService { + + /** + * Handles REST API calls that contain RPC requests to Device and pushes them to Rule Engine. + * Schedules the timeout for the RPC call based on the {@link ToDeviceRpcRequest} + * + * @param request the RPC request + * @param responseConsumer the consumer of the RPC response + */ + void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer); + + /** + * Handles the RPC response from the Rule Engine. + * + * @param response the RPC response + */ + void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response); + + /** + * Forwards the RPC request from Rule Engine to Device Actor + * + * @param request the RPC request message + */ + void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg request); + + /** + * Handles the RPC response from the Device Actor (Transport). + * + * @param response the RPC response + */ + void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java new file mode 100644 index 0000000000..3d4f23a796 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import org.thingsboard.rule.engine.api.RuleEngineRpcService; + +/** + * Created by ashvayka on 16.04.18. + */ +public interface TbRuleEngineDeviceRpcService extends RuleEngineRpcService { + + /** + * Handles the RPC response from the Device Actor (Transport). + * + * @param response the RPC response + */ + void processRpcResponseFromDevice(FromDeviceRpcResponse response); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java index ba537aed25..3cfe9bc1f4 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java @@ -22,11 +22,8 @@ import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import java.util.Optional; - /** * Created by ashvayka on 16.04.18. */ @@ -35,7 +32,7 @@ import java.util.Optional; public class ToDeviceRpcRequestActorMsg implements ToDeviceActorNotificationMsg { @Getter - private final ServerAddress serverAddress; + private final String serviceId; @Getter private final ToDeviceRpcRequest msg; diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index 31827f542e..daa74d013f 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractJsInvokeService implements JsInvokeService { protected Map scriptIdToNameMap = new ConcurrentHashMap<>(); - protected Map blackListedFunctions = new ConcurrentHashMap<>(); + protected Map blackListedFunctions = new ConcurrentHashMap<>(); @Override public ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames) { @@ -78,25 +78,53 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract int getMaxErrors(); + protected abstract long getMaxBlacklistDuration(); + protected void onScriptExecutionError(UUID scriptId) { - blackListedFunctions.computeIfAbsent(scriptId, key -> new AtomicInteger(0)).incrementAndGet(); + blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); } private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { - switch (scriptType) { - case RULE_NODE_SCRIPT: - return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); - default: - throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); + if (scriptType == JsScriptType.RULE_NODE_SCRIPT) { + return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); } + throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); } private boolean isBlackListed(UUID scriptId) { - if (blackListedFunctions.containsKey(scriptId)) { - AtomicInteger errorCount = blackListedFunctions.get(scriptId); - return errorCount.get() >= getMaxErrors(); + BlackListInfo errorCount = blackListedFunctions.get(scriptId); + if (errorCount != null) { + if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { + blackListedFunctions.remove(scriptId); + return false; + } else { + return errorCount.get() >= getMaxErrors(); + } } else { return false; } } + + private class BlackListInfo { + private final AtomicInteger counter; + private long expirationTime; + + private BlackListInfo() { + this.counter = new AtomicInteger(0); + } + + public int get() { + return counter.get(); + } + + public int incrementAndGet() { + int result = counter.incrementAndGet(); + expirationTime = System.currentTimeMillis() + getMaxBlacklistDuration(); + return result; + } + + public long getExpirationTime() { + return expirationTime; + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index 49a6304ebb..a9acfa4f42 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -55,8 +55,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer private final AtomicInteger jsEvalMsgs = new AtomicInteger(0); private final AtomicInteger jsFailedMsgs = new AtomicInteger(0); private final AtomicInteger jsTimeoutMsgs = new AtomicInteger(0); - private final FutureCallback evalCallback = new JsStatCallback(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); - private final FutureCallback invokeCallback = new JsStatCallback(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); @Autowired @Getter diff --git a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java index 909565609c..66a14cc827 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java @@ -20,6 +20,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import java.util.concurrent.TimeUnit; + @Slf4j @ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "local", matchIfMissing = true) @Service @@ -37,6 +39,9 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { @Value("${js.local.max_errors}") private int maxErrors; + @Value("${js.local.max_black_list_duration_sec:60}") + private int maxBlackListDurationSec; + @Override protected boolean useJsSandbox() { return useJsSandbox; @@ -56,4 +61,9 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } + + @Override + protected long getMaxBlacklistDuration() { + return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 1a218b119b..8d1f9d662a 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -23,15 +23,13 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaRequestTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -39,42 +37,25 @@ import javax.annotation.PreDestroy; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -@ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "remote", matchIfMissing = true) +@ConditionalOnExpression("'${js.evaluator:null}'=='remote' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine')") @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { - @Autowired - private TbNodeIdProvider nodeIdProvider; - - @Autowired - private TbKafkaSettings kafkaSettings; - - @Value("${js.remote.request_topic}") - private String requestTopic; - - @Value("${js.remote.response_topic_prefix}") - private String responseTopicPrefix; - - @Value("${js.remote.max_pending_requests}") - private long maxPendingRequests; - @Value("${js.remote.max_requests_timeout}") private long maxRequestsTimeout; - @Value("${js.remote.response_poll_interval}") - private int responsePollDuration; - - @Value("${js.remote.response_auto_commit_interval}") - private int autoCommitInterval; - @Getter @Value("${js.remote.max_errors}") private int maxErrors; + @Value("${js.remote.max_black_list_duration_sec:60}") + private int maxBlackListDurationSec; + @Value("${js.remote.stats.enabled:false}") private boolean statsEnabled; @@ -99,42 +80,20 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } - private TbKafkaRequestTemplate kafkaTemplate; + @Autowired + private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; + private Map scriptIdToBodysMap = new ConcurrentHashMap<>(); @PostConstruct public void init() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-js-invoke-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(requestTopic); - requestBuilder.encoder(new RemoteJsRequestEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(responseTopicPrefix + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("js-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - responseBuilder.autoCommitIntervalMs(autoCommitInterval); - responseBuilder.decoder(new RemoteJsResponseDecoder()); - responseBuilder.requestIdExtractor((response) -> new UUID(response.getRequestIdMSB(), response.getRequestIdLSB())); - - TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder - builder = TbKafkaRequestTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(maxPendingRequests); - builder.maxRequestTimeout(maxRequestsTimeout); - builder.pollInterval(responsePollDuration); - kafkaTemplate = builder.build(); - kafkaTemplate.init(); + requestTemplate.init(); } @PreDestroy public void destroy() { - if (kafkaTemplate != null) { - kafkaTemplate.stop(); + if (requestTemplate != null) { + requestTemplate.stop(); } } @@ -151,11 +110,12 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .build(); log.trace("Post compile request for scriptId [{}]", scriptId); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback>() { @Override - public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { + public void onSuccess(@Nullable TbProtoQueueMsg result) { kafkaEvalMsgs.incrementAndGet(); } @@ -168,7 +128,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }, MoreExecutors.directExecutor()); return Futures.transform(future, response -> { - JsInvokeProtos.JsCompileResponse compilationResult = response.getCompileResponse(); + JsInvokeProtos.JsCompileResponse compilationResult = response.getValue().getCompileResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); if (compilationResult.getSuccess()) { scriptIdToNameMap.put(scriptId, functionName); @@ -202,16 +162,17 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setInvokeRequest(jsRequestBuilder.build()) .build(); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback>() { @Override - public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { + public void onSuccess(@Nullable TbProtoQueueMsg result) { kafkaInvokeMsgs.incrementAndGet(); } @Override public void onFailure(Throwable t) { + onScriptExecutionError(scriptId); if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { kafkaTimeoutMsgs.incrementAndGet(); } @@ -219,10 +180,11 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }, MoreExecutors.directExecutor()); return Futures.transform(future, response -> { - JsInvokeProtos.JsInvokeResponse invokeResult = response.getInvokeResponse(); + JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse(); if (invokeResult.getSuccess()) { return invokeResult.getResult(); } else { + onScriptExecutionError(scriptId); log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); throw new RuntimeException(invokeResult.getErrorDetails()); } @@ -240,8 +202,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setReleaseRequest(jsRequest) .build(); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); - JsInvokeProtos.RemoteJsResponse response = future.get(); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + JsInvokeProtos.RemoteJsResponse response = future.get().getValue(); JsInvokeProtos.JsReleaseResponse compilationResult = response.getReleaseResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); @@ -252,4 +214,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } + @Override + protected long getMaxBlacklistDuration() { + return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java index d07a2490ba..51ecba27b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java @@ -17,19 +17,20 @@ package org.thingsboard.server.service.script; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsRequestEncoder implements TbKafkaEncoder { +public class RemoteJsRequestEncoder implements TbKafkaEncoder> { @Override - public byte[] encode(JsInvokeProtos.RemoteJsRequest value) { + public byte[] encode(TbProtoQueueMsg value) { try { - return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); + return JsonFormat.printer().print(value.getValue()).getBytes(StandardCharsets.UTF_8); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java index 7a3876fb91..621d2b05d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java @@ -16,8 +16,10 @@ package org.thingsboard.server.service.script; import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -25,12 +27,12 @@ import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsResponseDecoder implements TbKafkaDecoder { +public class RemoteJsResponseDecoder implements TbKafkaDecoder> { @Override - public JsInvokeProtos.RemoteJsResponse decode(byte[] data) throws IOException { + public TbProtoQueueMsg decode(TbQueueMsg msg) throws IOException { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(data, StandardCharsets.UTF_8), builder); - return builder.build(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index ef5d4716cb..110f410fad 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -95,7 +95,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S String newData = data != null ? data : msg.getData(); TbMsgMetaData newMetadata = metadata != null ? new TbMsgMetaData(metadata) : msg.getMetaData().copy(); String newMessageType = !StringUtils.isEmpty(messageType) ? messageType : msg.getType(); - return new TbMsg(msg.getId(), newMessageType, msg.getOriginator(), newMetadata, newData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); + return TbMsg.transformMsg(msg, newMessageType, msg.getOriginator(), newMetadata, newData); } catch (Throwable th) { th.printStackTrace(); throw new RuntimeException("Failed to unbind message data from javascript result", th); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index e359befb30..572c3a6ed0 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -46,6 +46,7 @@ import ua_parser.Client; import java.util.UUID; + @Component @Slf4j public class RestAuthenticationProvider implements AuthenticationProvider { diff --git a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java index c58a664790..c13cb5529a 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.security.device; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsFilter; @@ -24,16 +23,21 @@ import org.thingsboard.server.common.transport.auth.DeviceAuthResult; import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.TbCoreComponent; @Service +@TbCoreComponent @Slf4j public class DefaultDeviceAuthService implements DeviceAuthService { - @Autowired - DeviceService deviceService; + private final DeviceService deviceService; - @Autowired - DeviceCredentialsService deviceCredentialsService; + private final DeviceCredentialsService deviceCredentialsService; + + public DefaultDeviceAuthService(DeviceService deviceService, DeviceCredentialsService deviceCredentialsService) { + this.deviceService = deviceService; + this.deviceCredentialsService = deviceCredentialsService; + } @Override public DeviceAuthResult process(DeviceCredentialsFilter credentialsFilter) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java index 40d19da764..5412ed6def 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java @@ -15,11 +15,9 @@ */ package org.thingsboard.server.service.security.permission; -import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; public interface AccessControlService { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java index 9777af7b8e..a8c5dc13ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java @@ -16,20 +16,13 @@ package org.thingsboard.server.service.security.permission; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.Authority; -import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.*; diff --git a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java index 952c63c9b9..1b9d566206 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java +++ b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java @@ -21,9 +21,9 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.Collections; -import java.util.UUID; import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; @@ -31,6 +31,7 @@ import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; * Created by ashvayka on 29.10.18. */ @Service +@TbCoreComponent @Slf4j public class DefaultDeviceSessionCacheService implements DeviceSessionCacheService { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 18726e3a5a..77a932e10d 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.state; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.util.concurrent.FutureCallback; @@ -23,16 +22,13 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; @@ -49,16 +45,18 @@ import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -69,7 +67,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -89,8 +86,8 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; * Created by ashvayka on 01.05.18. */ @Service +@TbCoreComponent @Slf4j -//TODO: refactor to use page links as cursor and not fetch all public class DefaultDeviceStateService implements DeviceStateService { private static final ObjectMapper json = new ObjectMapper(); @@ -104,31 +101,15 @@ public class DefaultDeviceStateService implements DeviceStateService { public static final List PERSISTENT_ATTRIBUTES = Arrays.asList(ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT); - @Autowired - private TenantService tenantService; - - @Autowired - private DeviceService deviceService; + private final TenantService tenantService; + private final DeviceService deviceService; + private final AttributesService attributesService; + private final TimeseriesService tsService; + private final TbClusterService clusterService; + private final PartitionService partitionService; - @Autowired - private AttributesService attributesService; - - @Autowired - private TimeseriesService tsService; - - @Autowired - @Lazy - private ActorService actorService; - - @Autowired private TelemetrySubscriptionService tsSubService; - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService clusterRpcService; - @Value("${state.defaultInactivityTimeoutInSec}") @Getter private long defaultInactivityTimeoutInSec; @@ -148,18 +129,32 @@ public class DefaultDeviceStateService implements DeviceStateService { private volatile boolean clusterUpdatePending = false; private ListeningScheduledExecutorService queueExecutor; - private ConcurrentMap> tenantDevices = new ConcurrentHashMap<>(); - private ConcurrentMap deviceStates = new ConcurrentHashMap<>(); - private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); - private ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); + private final ConcurrentMap> partitionedDevices = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceStates = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); + + public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, + AttributesService attributesService, TimeseriesService tsService, + TbClusterService clusterService, PartitionService partitionService) { + this.tenantService = tenantService; + this.deviceService = deviceService; + this.attributesService = attributesService; + this.tsService = tsService; + this.clusterService = clusterService; + this.partitionService = partitionService; + } + + @Autowired + public void setTsSubService(TelemetrySubscriptionService tsSubService) { + this.tsSubService = tsSubService; + } @PostConstruct public void init() { // Should be always single threaded due to absence of locks. queueExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-state"))); - queueExecutor.submit(this::initStateFromDB); queueExecutor.scheduleAtFixedRate(this::updateState, new Random().nextInt(defaultStateCheckIntervalInSec), defaultStateCheckIntervalInSec, TimeUnit.SECONDS); - //TODO: schedule persistence in v2.1; } @PreDestroy @@ -171,133 +166,210 @@ public class DefaultDeviceStateService implements DeviceStateService { @Override public void onDeviceAdded(Device device) { - queueExecutor.submit(() -> onDeviceAddedSync(device)); + sendDeviceEvent(device.getTenantId(), device.getId(), true, false, false); } @Override public void onDeviceUpdated(Device device) { - queueExecutor.submit(() -> onDeviceUpdatedSync(device)); + sendDeviceEvent(device.getTenantId(), device.getId(), false, true, false); } @Override - public void onDeviceConnect(DeviceId deviceId) { - queueExecutor.submit(() -> onDeviceConnectSync(deviceId)); + public void onDeviceDeleted(Device device) { + sendDeviceEvent(device.getTenantId(), device.getId(), false, false, true); } @Override - public void onDeviceActivity(DeviceId deviceId) { - deviceLastReportedActivity.put(deviceId, System.currentTimeMillis()); - queueExecutor.submit(() -> onDeviceActivitySync(deviceId)); + public void onDeviceConnect(DeviceId deviceId) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastConnectTime(ts); + pushRuleEngineMessage(stateData, CONNECT_EVENT); + save(deviceId, LAST_CONNECT_TIME, ts); + } } @Override - public void onDeviceDisconnect(DeviceId deviceId) { - queueExecutor.submit(() -> onDeviceDisconnectSync(deviceId)); + public void onDeviceActivity(DeviceId deviceId, long lastReportedActivity) { + deviceLastReportedActivity.put(deviceId, lastReportedActivity); + long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); + if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + DeviceState state = stateData.getState(); + stateData.getState().setLastActivityTime(lastReportedActivity); + stateData.getMetaData().putValue("scope", SERVER_SCOPE); + pushRuleEngineMessage(stateData, ACTIVITY_EVENT); + save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); + deviceLastSavedActivity.put(deviceId, lastReportedActivity); + if (!state.isActive()) { + state.setActive(true); + save(deviceId, ACTIVITY_STATE, state.isActive()); + } + } + } } @Override - public void onDeviceDeleted(Device device) { - queueExecutor.submit(() -> onDeviceDeleted(device.getTenantId(), device.getId())); + public void onDeviceDisconnect(DeviceId deviceId) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastDisconnectTime(ts); + pushRuleEngineMessage(stateData, DISCONNECT_EVENT); + save(deviceId, LAST_DISCONNECT_TIME, ts); + } } @Override public void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { - queueExecutor.submit(() -> onInactivityTimeoutUpdate(deviceId, inactivityTimeout)); - } - - @Override - public void onClusterUpdate() { - if (!clusterUpdatePending) { - clusterUpdatePending = true; - queueExecutor.submit(this::onClusterUpdateSync); + if (inactivityTimeout == 0L) { + return; + } + DeviceStateData stateData = deviceStates.get(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + DeviceState state = stateData.getState(); + state.setInactivityTimeout(inactivityTimeout); + boolean oldActive = state.isActive(); + state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); + if (!oldActive && state.isActive() || oldActive && !state.isActive()) { + save(deviceId, ACTIVITY_STATE, state.isActive()); + } } } @Override - public void onRemoteMsg(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.DeviceStateServiceMsgProto proto; + public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbCallback callback) { try { - proto = ClusterAPIProtos.DeviceStateServiceMsgProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); - if (proto.getDeleted()) { - queueExecutor.submit(() -> onDeviceDeleted(tenantId, deviceId)); - } else { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - if (proto.getAdded()) { - onDeviceAdded(device); - } else if (proto.getUpdated()) { - onDeviceUpdated(device); + TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); + if (proto.getDeleted()) { + onDeviceDeleted(tenantId, deviceId); + callback.onSuccess(); + } else { + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + if (proto.getAdded()) { + Futures.addCallback(fetchDeviceState(device), new FutureCallback() { + @Override + public void onSuccess(@Nullable DeviceStateData state) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, device.getId()); + if (partitionedDevices.containsKey(tpi)) { + addDeviceUsingState(tpi, state); + callback.onSuccess(); + } else { + log.warn("[{}][{}] Device belongs to external partition. Probably rebalancing is in progress. Topic: {}" + , tenantId, deviceId, tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Device belongs to external partition " + tpi.getFullTopicName() + "!")); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to register device to the state service", t); + callback.onFailure(t); + } + }, MoreExecutors.directExecutor()); + } else if (proto.getUpdated()) { + DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); + if (stateData != null) { + TbMsgMetaData md = new TbMsgMetaData(); + md.putValue("deviceName", device.getName()); + md.putValue("deviceType", device.getType()); + stateData.setMetaData(md); + } + } + } else { + //Device was probably deleted while message was in queue; + callback.onSuccess(); } } + } catch (Exception e) { + log.trace("Failed to process queue msg: [{}]", proto, e); + callback.onFailure(e); } } - private void onClusterUpdateSync() { - clusterUpdatePending = false; - List tenants = tenantService.findTenants(new TextPageLink(Integer.MAX_VALUE)).getData(); - for (Tenant tenant : tenants) { - List> fetchFutures = new ArrayList<>(); - TextPageLink pageLink = new TextPageLink(initFetchPackSize); - while (pageLink != null) { - TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); - pageLink = page.getNextPageLink(); - for (Device device : page.getData()) { - if (!routingService.resolveById(device.getId()).isPresent()) { - if (!deviceStates.containsKey(device.getId())) { - fetchFutures.add(fetchDeviceState(device)); - } - } else { - Set tenantDeviceSet = tenantDevices.get(tenant.getId()); - if (tenantDeviceSet != null) { - tenantDeviceSet.remove(device.getId()); - } - deviceStates.remove(device.getId()); - deviceLastReportedActivity.remove(device.getId()); - deviceLastSavedActivity.remove(device.getId()); - } - } - try { - Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); - } catch (InterruptedException | ExecutionException e) { - log.warn("Failed to init device state service from DB", e); + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + synchronized (this) { + if (!clusterUpdatePending) { + clusterUpdatePending = true; + queueExecutor.submit(() -> { + clusterUpdatePending = false; + initStateFromDB(partitionChangeEvent.getPartitions()); + }); } } } } - private void initStateFromDB() { + private void initStateFromDB(Set partitions) { try { + Set addedPartitions = new HashSet<>(partitions); + addedPartitions.removeAll(partitionedDevices.keySet()); + + Set removedPartitions = new HashSet<>(partitionedDevices.keySet()); + removedPartitions.removeAll(partitions); + + // We no longer manage current partition of devices; + removedPartitions.forEach(partition -> { + Set devices = partitionedDevices.remove(partition); + devices.forEach(deviceId -> { + deviceStates.remove(deviceId); + deviceLastReportedActivity.remove(deviceId); + deviceLastSavedActivity.remove(deviceId); + }); + }); + + addedPartitions.forEach(tpi -> partitionedDevices.computeIfAbsent(tpi, key -> ConcurrentHashMap.newKeySet())); + + //TODO 3.0: replace this dummy search with new functionality to search by partitions using SQL capabilities. + // Adding only devices that are in new partitions List tenants = tenantService.findTenants(new TextPageLink(Integer.MAX_VALUE)).getData(); for (Tenant tenant : tenants) { - List> fetchFutures = new ArrayList<>(); TextPageLink pageLink = new TextPageLink(initFetchPackSize); while (pageLink != null) { + List> fetchFutures = new ArrayList<>(); TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.getNextPageLink(); for (Device device : page.getData()) { - if (!routingService.resolveById(device.getId()).isPresent()) { - fetchFutures.add(fetchDeviceState(device)); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), device.getId()); + if (addedPartitions.contains(tpi)) { + ListenableFuture future = Futures.transform(fetchDeviceState(device), new Function() { + @Nullable + @Override + public Void apply(@Nullable DeviceStateData state) { + if (state != null) { + addDeviceUsingState(tpi, state); + } + return null; + } + }, MoreExecutors.directExecutor()); + fetchFutures.add(future); } } try { - Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); + Futures.successfulAsList(fetchFutures).get(); } catch (InterruptedException | ExecutionException e) { log.warn("Failed to init device state service from DB", e); } } } + log.info("Managing following partitions:"); + partitionedDevices.forEach((tpi, devices) -> { + log.info("[{}]: {} devices", tpi.getFullTopicName(), devices.size()); + }); } catch (Throwable t) { log.warn("Failed to init device states from DB", t); } } - private void addDeviceUsingState(DeviceStateData state) { - tenantDevices.computeIfAbsent(state.getTenantId(), id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); + private void addDeviceUsingState(TopicPartitionInfo tpi, DeviceStateData state) { + partitionedDevices.computeIfAbsent(tpi, id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); deviceStates.put(state.getDeviceId(), state); } @@ -325,103 +397,24 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private void onDeviceConnectSync(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastConnectTime(ts); - pushRuleEngineMessage(stateData, CONNECT_EVENT); - save(deviceId, LAST_CONNECT_TIME, ts); - } - } - - private void onDeviceDisconnectSync(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastDisconnectTime(ts); - pushRuleEngineMessage(stateData, DISCONNECT_EVENT); - save(deviceId, LAST_DISCONNECT_TIME, ts); - } - } - - private void onDeviceActivitySync(DeviceId deviceId) { - long lastReportedActivity = deviceLastReportedActivity.getOrDefault(deviceId, 0L); - long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); - if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - DeviceState state = stateData.getState(); - stateData.getState().setLastActivityTime(lastReportedActivity); - stateData.getMetaData().putValue("scope", SERVER_SCOPE); - pushRuleEngineMessage(stateData, ACTIVITY_EVENT); - save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); - deviceLastSavedActivity.put(deviceId, lastReportedActivity); - if (!state.isActive()) { - state.setActive(true); - save(deviceId, ACTIVITY_STATE, state.isActive()); - } - } - } - } - private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) { DeviceStateData deviceStateData = deviceStates.get(deviceId); if (deviceStateData == null) { - if (!routingService.resolveById(deviceId).isPresent()) { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - try { - deviceStateData = fetchDeviceState(device).get(); - deviceStates.putIfAbsent(deviceId, deviceStateData); - } catch (InterruptedException | ExecutionException e) { - log.debug("[{}] Failed to fetch device state!", deviceId, e); - } + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + try { + deviceStateData = fetchDeviceState(device).get(); + deviceStates.putIfAbsent(deviceId, deviceStateData); + } catch (InterruptedException | ExecutionException e) { + log.debug("[{}] Failed to fetch device state!", deviceId, e); } } } return deviceStateData; } - private void onInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { - if (inactivityTimeout == 0L) { - return; - } - DeviceStateData stateData = deviceStates.get(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - DeviceState state = stateData.getState(); - state.setInactivityTimeout(inactivityTimeout); - boolean oldActive = state.isActive(); - state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); - if (!oldActive && state.isActive() || oldActive && !state.isActive()) { - save(deviceId, ACTIVITY_STATE, state.isActive()); - } - } - } - - private void onDeviceAddedSync(Device device) { - Optional address = routingService.resolveById(device.getId()); - if (!address.isPresent()) { - Futures.addCallback(fetchDeviceState(device), new FutureCallback() { - @Override - public void onSuccess(@Nullable DeviceStateData state) { - addDeviceUsingState(state); - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to register device to the state service", t); - } - }, MoreExecutors.directExecutor()); - } else { - sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); - } - } - - private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, ServerAddress address, boolean added, boolean updated, boolean deleted) { - log.trace("[{}][{}] Device is monitored on other server: {}", tenantId, deviceId, address); - ClusterAPIProtos.DeviceStateServiceMsgProto.Builder builder = ClusterAPIProtos.DeviceStateServiceMsgProto.newBuilder(); + private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, boolean added, boolean updated, boolean deleted) { + TransportProtos.DeviceStateServiceMsgProto.Builder builder = TransportProtos.DeviceStateServiceMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setDeviceIdMSB(deviceId.getId().getMostSignificantBits()); @@ -429,40 +422,17 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); - } - - private void onDeviceUpdatedSync(Device device) { - Optional address = routingService.resolveById(device.getId()); - if (!address.isPresent()) { - DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); - if (stateData != null) { - TbMsgMetaData md = new TbMsgMetaData(); - md.putValue("deviceName", device.getName()); - md.putValue("deviceType", device.getType()); - stateData.setMetaData(md); - } - } else { - sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); - } + TransportProtos.DeviceStateServiceMsgProto msg = builder.build(); + clusterService.pushMsgToCore(tenantId, deviceId, TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build(), null); } private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) { - Optional address = routingService.resolveById(deviceId); - if (!address.isPresent()) { - deviceStates.remove(deviceId); - deviceLastReportedActivity.remove(deviceId); - deviceLastSavedActivity.remove(deviceId); - Set deviceIds = tenantDevices.get(tenantId); - if (deviceIds != null) { - deviceIds.remove(deviceId); - if (deviceIds.isEmpty()) { - tenantDevices.remove(tenantId); - } - } - } else { - sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); - } + deviceStates.remove(deviceId); + deviceLastReportedActivity.remove(deviceId); + deviceLastSavedActivity.remove(deviceId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); + Set deviceIdSet = partitionedDevices.get(tpi); + deviceIdSet.remove(deviceId); } private ListenableFuture fetchDeviceState(Device device) { @@ -523,10 +493,9 @@ public class DefaultDeviceStateService implements DeviceStateService { private void pushRuleEngineMessage(DeviceStateData stateData, String msgType) { DeviceState state = stateData.getState(); try { - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON - , json.writeValueAsString(state) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(stateData.getDeviceId(), new ServiceToRuleEngineMsg(stateData.getTenantId(), tbMsg))); + TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON + , json.writeValueAsString(state)); + clusterService.pushMsgToRuleEngine(stateData.getTenantId(), stateData.getDeviceId(), tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } @@ -554,7 +523,7 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private class AttributeSaveCallback implements FutureCallback { + private static class AttributeSaveCallback implements FutureCallback { private final DeviceId deviceId; private final String key; private final Object value; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 8dbe7579e3..1665a27b84 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.service.state; +import org.springframework.context.ApplicationListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.msg.queue.TbCallback; /** * Created by ashvayka on 01.05.18. */ -public interface DeviceStateService { +public interface DeviceStateService extends ApplicationListener { void onDeviceAdded(Device device); @@ -32,13 +35,12 @@ public interface DeviceStateService { void onDeviceConnect(DeviceId deviceId); - void onDeviceActivity(DeviceId deviceId); + void onDeviceActivity(DeviceId deviceId, long lastReportedActivityTime); void onDeviceDisconnect(DeviceId deviceId); void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout); - void onClusterUpdate(); + void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbCallback bytes); - void onRemoteMsg(ServerAddress serverAddress, byte[] bytes); } diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java new file mode 100644 index 0000000000..a506b87ab0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -0,0 +1,141 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.stats; + +import com.google.common.util.concurrent.FutureCallback; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; + +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +@TbRuleEngineComponent +@Service +@Slf4j +public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { + + public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; + public static final FutureCallback CALLBACK = new FutureCallback() { + @Override + public void onSuccess(@Nullable Void result) { + + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to persist statistics", t); + } + }; + + private final TbServiceInfoProvider serviceInfoProvider; + private final TelemetrySubscriptionService tsService; + private final Lock lock = new ReentrantLock(); + private final AssetService assetService; + private final ConcurrentMap tenantQueueAssets; + + public DefaultRuleEngineStatisticsService(TelemetrySubscriptionService tsService, TbServiceInfoProvider serviceInfoProvider, AssetService assetService) { + this.tsService = tsService; + this.serviceInfoProvider = serviceInfoProvider; + this.assetService = assetService; + this.tenantQueueAssets = new ConcurrentHashMap<>(); + } + + @Override + public void reportQueueStats(long ts, TbRuleEngineConsumerStats ruleEngineStats) { + String queueName = ruleEngineStats.getQueueName(); + ruleEngineStats.getTenantStats().forEach((id, stats) -> { + TenantId tenantId = new TenantId(id); + try { + AssetId serviceAssetId = getServiceAssetId(tenantId, queueName); + if (stats.getTotalMsgCounter().get() > 0) { + List tsList = stats.getCounters().entrySet().stream() + .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) + .collect(Collectors.toList()); + if (!tsList.isEmpty()) { + tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + } + } + } catch (DataValidationException e) { + if (!e.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { + throw e; + } + } + }); + ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { + TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); + try { + tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + } catch (DataValidationException e2) { + if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { + throw e2; + } + } + }); + ruleEngineStats.reset(); + } + + private AssetId getServiceAssetId(TenantId tenantId, String queueName) { + TenantQueueKey key = new TenantQueueKey(tenantId, queueName); + AssetId assetId = tenantQueueAssets.get(key); + if (assetId == null) { + lock.lock(); + try { + assetId = tenantQueueAssets.get(key); + if (assetId == null) { + Asset asset = assetService.findAssetByTenantIdAndName(tenantId, queueName + "_" + serviceInfoProvider.getServiceId()); + if (asset == null) { + asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName(queueName + "_" + serviceInfoProvider.getServiceId()); + asset.setType(TB_SERVICE_QUEUE); + asset = assetService.saveAsset(asset); + } + assetId = asset.getId(); + tenantQueueAssets.put(key, assetId); + } + } finally { + lock.unlock(); + } + } + return assetId; + } + + @Data + private static class TenantQueueKey { + private final TenantId tenantId; + private final String queueName; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java new file mode 100644 index 0000000000..88573844c9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.stats; + +import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; + +public interface RuleEngineStatisticsService { + + void reportQueueStats(long ts, TbRuleEngineConsumerStats stats); +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java new file mode 100644 index 0000000000..67c81ac06f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -0,0 +1,380 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.common.util.DonAsynchron; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos.*; +import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateValueListProto; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.state.DefaultDeviceStateService; +import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Function; +import java.util.function.Predicate; + +@Slf4j +@TbCoreComponent +@Service +public class DefaultSubscriptionManagerService implements SubscriptionManagerService { + + @Autowired + private AttributesService attrService; + + @Autowired + private TimeseriesService tsService; + + @Autowired + private PartitionService partitionService; + + @Autowired + private TbServiceInfoProvider serviceInfoProvider; + + @Autowired + private TbQueueProducerProvider producerProvider; + + @Autowired + private TbLocalSubscriptionService localSubscriptionService; + + @Autowired + private DeviceStateService deviceStateService; + + @Autowired + private TbClusterService clusterService; + + private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); + private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); + private final ConcurrentMap> partitionedSubscriptions = new ConcurrentHashMap<>(); + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); + + private ExecutorService tsCallBackExecutor; + private String serviceId; + private TbQueueProducer> toCoreNotificationsProducer; + + @PostConstruct + public void initExecutor() { + tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); + serviceId = serviceInfoProvider.getServiceId(); + toCoreNotificationsProducer = producerProvider.getTbCoreNotificationsMsgProducer(); + } + + @PreDestroy + public void shutdownExecutor() { + if (tsCallBackExecutor != null) { + tsCallBackExecutor.shutdownNow(); + } + } + + @Override + public void addSubscription(TbSubscription subscription, TbCallback callback) { + log.trace("[{}][{}][{}] Registering remote subscription for entity [{}]", + subscription.getServiceId(), subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + partitionedSubscriptions.computeIfAbsent(tpi, k -> ConcurrentHashMap.newKeySet()).add(subscription); + callback.onSuccess(); + } else { + log.warn("[{}][{}] Entity belongs to external partition. Probably rebalancing is in progress. Topic: {}" + , subscription.getTenantId(), subscription.getEntityId(), tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Entity belongs to external partition " + tpi.getFullTopicName() + "!")); + } + boolean newSubscription = subscriptionsByEntityId + .computeIfAbsent(subscription.getEntityId(), k -> ConcurrentHashMap.newKeySet()).add(subscription); + subscriptionsByWsSessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()).put(subscription.getSubscriptionId(), subscription); + if (newSubscription) { + switch (subscription.getType()) { + case TIMESERIES: + handleNewTelemetrySubscription((TbTimeseriesSubscription) subscription); + break; + case ATTRIBUTES: + handleNewAttributeSubscription((TbAttributeSubscription) subscription); + break; + } + } + } + + @Override + public void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback) { + log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); + Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); + if (sessionSubscriptions != null) { + TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); + if (subscription != null) { + removeSubscriptionFromEntityMap(subscription); + removeSubscriptionFromPartitionMap(subscription); + if (sessionSubscriptions.isEmpty()) { + subscriptionsByWsSessionId.remove(sessionId); + } + } else { + log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); + } + } else { + log.debug("[{}] No session subscriptions found!", sessionId); + } + callback.onSuccess(); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + Set removedPartitions = new HashSet<>(currentPartitions); + removedPartitions.removeAll(partitionChangeEvent.getPartitions()); + + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); + + // We no longer manage current partition of devices; + removedPartitions.forEach(partition -> { + Set subs = partitionedSubscriptions.remove(partition); + if (subs != null) { + subs.forEach(this::removeSubscriptionFromEntityMap); + } + }); + } + } + + @Override + public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback) { + onLocalSubUpdate(entityId, + s -> { + if (TbSubscriptionType.TIMESERIES.equals(s.getType())) { + return (TbTimeseriesSubscription) s; + } else { + return null; + } + }, s -> true, s -> { + List subscriptionUpdate = null; + for (TsKvEntry kv : ts) { + if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(kv); + } + } + return subscriptionUpdate; + }); + callback.onSuccess(); + } + + @Override + public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback) { + onLocalSubUpdate(entityId, + s -> { + if (TbSubscriptionType.ATTRIBUTES.equals(s.getType())) { + return (TbAttributeSubscription) s; + } else { + return null; + } + }, + s -> (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope().name())), + s -> { + List subscriptionUpdate = null; + for (AttributeKvEntry kv : attributes) { + if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); + } + } + return subscriptionUpdate; + }); + if (entityId.getEntityType() == EntityType.DEVICE) { + if (TbAttributeSubscriptionScope.SERVER_SCOPE.name().equalsIgnoreCase(scope)) { + for (AttributeKvEntry attribute : attributes) { + if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { + deviceStateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); + } + } + } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope)) { + clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate(tenantId, + new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)) + , null); + } + } + callback.onSuccess(); + } + + private void onLocalSubUpdate(EntityId entityId, + Function castFunction, + Predicate filterFunction, + Function> processFunction) { + Set entitySubscriptions = subscriptionsByEntityId.get(entityId); + if (entitySubscriptions != null) { + entitySubscriptions.stream().map(castFunction).filter(Objects::nonNull).filter(filterFunction).forEach(s -> { + List subscriptionUpdate = processFunction.apply(s); + if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { + if (serviceId.equals(s.getServiceId())) { + SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); + localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbCallback.EMPTY); + } else { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId()); + toCoreNotificationsProducer.send(tpi, toProto(s, subscriptionUpdate), null); + } + } + }); + } else { + log.debug("[{}] No device subscriptions to process!", entityId); + } + } + + private boolean isInTimeRange(TbTimeseriesSubscription subscription, long kvTime) { + return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) + && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); + } + + private void removeSubscriptionFromEntityMap(TbSubscription sub) { + Set entitySubSet = subscriptionsByEntityId.get(sub.getEntityId()); + if (entitySubSet != null) { + entitySubSet.remove(sub); + if (entitySubSet.isEmpty()) { + subscriptionsByEntityId.remove(sub.getEntityId()); + } + } + } + + private void removeSubscriptionFromPartitionMap(TbSubscription sub) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, sub.getTenantId(), sub.getEntityId()); + Set subs = partitionedSubscriptions.get(tpi); + if (subs != null) { + subs.remove(sub); + } + } + + private void handleNewAttributeSubscription(TbAttributeSubscription subscription) { + log.trace("[{}][{}][{}] Processing remote attribute subscription for entity [{}]", + serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + + final Map keyStates = subscription.getKeyStates(); + DonAsynchron.withCallback(attrService.find(subscription.getTenantId(), subscription.getEntityId(), DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { + List missedUpdates = new ArrayList<>(); + values.forEach(latestEntry -> { + if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { + missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); + } + }); + if (!missedUpdates.isEmpty()) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); + toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); + } + }, + e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); + } + + private void handleNewTelemetrySubscription(TbTimeseriesSubscription subscription) { + log.trace("[{}][{}][{}] Processing remote telemetry subscription for entity [{}]", + serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + + long curTs = System.currentTimeMillis(); + List queries = new ArrayList<>(); + subscription.getKeyStates().forEach((key, value) -> { + if (curTs > value) { + long startTs = subscription.getStartTime() > 0 ? Math.max(subscription.getStartTime(), value + 1L) : (value + 1L); + long endTs = subscription.getEndTime() > 0 ? Math.min(subscription.getEndTime(), curTs) : curTs; + queries.add(new BaseReadTsKvQuery(key, startTs, endTs, 0, 1000, Aggregation.NONE)); + } + }); + if (!queries.isEmpty()) { + DonAsynchron.withCallback(tsService.findAll(subscription.getTenantId(), subscription.getEntityId(), queries), + missedUpdates -> { + if (missedUpdates != null && !missedUpdates.isEmpty()) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); + toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); + } + }, + e -> log.error("Failed to fetch missed updates.", e), + tsCallBackExecutor); + } + } + + private TbProtoQueueMsg toProto(TbSubscription subscription, List updates) { + TbSubscriptionUpdateProto.Builder builder = TbSubscriptionUpdateProto.newBuilder(); + + builder.setSessionId(subscription.getSessionId()); + builder.setSubscriptionId(subscription.getSubscriptionId()); + + Map> data = new TreeMap<>(); + for (TsKvEntry tsEntry : updates) { + List values = data.computeIfAbsent(tsEntry.getKey(), k -> new ArrayList<>()); + Object[] value = new Object[2]; + value[0] = tsEntry.getTs(); + value[1] = tsEntry.getValueAsString(); + values.add(value); + } + + data.forEach((key, value) -> { + TbSubscriptionUpdateValueListProto.Builder dataBuilder = TbSubscriptionUpdateValueListProto.newBuilder(); + dataBuilder.setKey(key); + value.forEach(v -> { + Object[] array = (Object[]) v; + dataBuilder.addTs((long) array[0]); + dataBuilder.addValue((String) array[1]); + }); + builder.addData(dataBuilder.build()); + }); + + ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setToLocalSubscriptionServiceMsg( + LocalSubscriptionServiceMsgProto.newBuilder().setSubUpdate(builder.build()).build()) + .build(); + return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java new file mode 100644 index 0000000000..d57067fa28 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -0,0 +1,228 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@Slf4j +@TbCoreComponent +@Service +public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionService { + + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); + private final Map> subscriptionsBySessionId = new ConcurrentHashMap<>(); + + @Autowired + private TelemetryWebSocketService wsService; + + @Autowired + private EntityViewService entityViewService; + + @Autowired + private PartitionService partitionService; + + @Autowired + private TbClusterService clusterService; + + @Autowired + @Lazy + private SubscriptionManagerService subscriptionManagerService; + + private ExecutorService wsCallBackExecutor; + + @PostConstruct + public void initExecutor() { + wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); + } + + @PreDestroy + public void shutdownExecutor() { + if (wsCallBackExecutor != null) { + wsCallBackExecutor.shutdownNow(); + } + } + + @Override + @EventListener(PartitionChangeEvent.class) + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); + } + } + + @Override + @EventListener(ClusterTopologyChangeEvent.class) + public void onApplicationEvent(ClusterTopologyChangeEvent event) { + if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) { + /* + * If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again. + * Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart. + * Although this is resource consuming operation, it is cheaper than sending ping/pong commands periodically + * It is also cheaper then caching the subscriptions by entity id and then lookup of those caches every time we have new telemetry in SubscriptionManagerService. + * Even if we cache locally the list of active subscriptions by entity id, it is still time consuming operation to get them from cache + * Since number of subscriptions is usually much less then number of devices that are pushing data. + */ + subscriptionsBySessionId.values().forEach(map -> map.values() + .forEach(sub -> pushSubscriptionToManagerService(sub, false))); + } + } + + //TODO 3.1: replace null callbacks with callbacks from websocket service. + @Override + public void addSubscription(TbSubscription subscription) { + EntityId entityId = subscription.getEntityId(); + // Telemetry subscription on Entity Views are handled differently, because we need to allow only certain keys and time ranges; + if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TbSubscriptionType.TIMESERIES.equals(subscription.getType())) { + subscription = resolveEntityViewSubscription((TbTimeseriesSubscription) subscription); + } + pushSubscriptionToManagerService(subscription, true); + registerSubscription(subscription); + } + + private void pushSubscriptionToManagerService(TbSubscription subscription, boolean pushToLocalService) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + // Subscription is managed on the same server; + if (pushToLocalService) { + subscriptionManagerService.addSubscription(subscription, TbCallback.EMPTY); + } + } else { + // Push to the queue; + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toNewSubscriptionProto(subscription); + clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); + } + } + + @Override + public void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback) { + TbSubscription subscription = subscriptionsBySessionId + .getOrDefault(sessionId, Collections.emptyMap()).get(update.getSubscriptionId()); + if (subscription != null) { + switch (subscription.getType()) { + case TIMESERIES: + TbTimeseriesSubscription tsSub = (TbTimeseriesSubscription) subscription; + update.getLatestValues().forEach((key, value) -> tsSub.getKeyStates().put(key, value)); + break; + case ATTRIBUTES: + TbAttributeSubscription attrSub = (TbAttributeSubscription) subscription; + update.getLatestValues().forEach((key, value) -> attrSub.getKeyStates().put(key, value)); + break; + } + wsService.sendWsMsg(sessionId, update); + } + callback.onSuccess(); + } + + @Override + public void cancelSubscription(String sessionId, int subscriptionId) { + log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); + Map sessionSubscriptions = subscriptionsBySessionId.get(sessionId); + if (sessionSubscriptions != null) { + TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); + if (subscription != null) { + if (sessionSubscriptions.isEmpty()) { + subscriptionsBySessionId.remove(sessionId); + } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + // Subscription is managed on the same server; + subscriptionManagerService.cancelSubscription(sessionId, subscriptionId, TbCallback.EMPTY); + } else { + // Push to the queue; + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toCloseSubscriptionProto(subscription); + clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); + } + } else { + log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); + } + } else { + log.debug("[{}] No session subscriptions found!", sessionId); + } + } + + @Override + public void cancelAllSessionSubscriptions(String sessionId) { + Map subscriptions = subscriptionsBySessionId.get(sessionId); + if (subscriptions != null) { + Set toRemove = new HashSet<>(subscriptions.keySet()); + toRemove.forEach(id -> cancelSubscription(sessionId, id)); + } + } + + private TbSubscription resolveEntityViewSubscription(TbTimeseriesSubscription subscription) { + EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(subscription.getEntityId().getId())); + + Map keyStates; + if (subscription.isAllKeys()) { + keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); + } else { + keyStates = subscription.getKeyStates().entrySet() + .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + return TbTimeseriesSubscription.builder() + .serviceId(subscription.getServiceId()) + .sessionId(subscription.getSessionId()) + .subscriptionId(subscription.getSubscriptionId()) + .tenantId(subscription.getTenantId()) + .entityId(entityView.getEntityId()) + .startTime(entityView.getStartTimeMs()) + .endTime(entityView.getEndTimeMs()) + .allKeys(false) + .keyStates(keyStates).build(); + } + + private void registerSubscription(TbSubscription subscription) { + Map sessionSubscriptions = subscriptionsBySessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()); + sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java new file mode 100644 index 0000000000..e20b707348 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.common.msg.queue.TbCallback; + +import java.util.List; + +public interface SubscriptionManagerService extends ApplicationListener { + + void addSubscription(TbSubscription subscription, TbCallback callback); + + void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback); + + void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback); + + void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java new file mode 100644 index 0000000000..83a86efefc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +public class TbAttributeSubscription extends TbSubscription { + + @Getter private final boolean allKeys; + @Getter private final Map keyStates; + @Getter private final TbAttributeSubscriptionScope scope; + + @Builder + public TbAttributeSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, + boolean allKeys, Map keyStates, TbAttributeSubscriptionScope scope) { + super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.ATTRIBUTES); + this.allKeys = allKeys; + this.keyStates = keyStates; + this.scope = scope; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java rename to application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java index 095e10f0b3..d23cc3581d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java @@ -13,12 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.discovery; +package org.thingsboard.server.service.subscription; -/** - * @author Andrew Shvayka - */ -public interface ServerInstanceService { +public enum TbAttributeSubscriptionScope { + + CLIENT_SCOPE, SHARED_SCOPE, SERVER_SCOPE - ServerInstance getSelf(); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java new file mode 100644 index 0000000000..58fba24250 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +public interface TbLocalSubscriptionService { + + void addSubscription(TbSubscription subscription); + + void cancelSubscription(String sessionId, int subscriptionId); + + void cancelAllSessionSubscriptions(String sessionId); + + void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback); + + void onApplicationEvent(PartitionChangeEvent event); + + void onApplicationEvent(ClusterTopologyChangeEvent event); +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java new file mode 100644 index 0000000000..22b37ff690 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; + +@Data +@AllArgsConstructor +public abstract class TbSubscription { + + private final String serviceId; + private final String sessionId; + private final int subscriptionId; + private final TenantId tenantId; + private final EntityId entityId; + private final TbSubscriptionType type; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TbSubscription that = (TbSubscription) o; + return subscriptionId == that.subscriptionId && + sessionId.equals(that.sessionId) && + tenantId.equals(that.tenantId) && + entityId.equals(that.entityId) && + type == that.type; + } + + @Override + public int hashCode() { + return Objects.hash(sessionId, subscriptionId, tenantId, entityId, type); + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java similarity index 82% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java rename to application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java index 4b00284c34..d7a4715460 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java @@ -13,11 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.service.subscription; -/** - * Created by ashvayka on 23.09.18. - */ -public enum ServerType { - CORE +public enum TbSubscriptionType { + TIMESERIES, ATTRIBUTES } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java new file mode 100644 index 0000000000..d0b4d713f9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java @@ -0,0 +1,247 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionKetStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; + +public class TbSubscriptionUtils { + + public static ToCoreMsg toNewSubscriptionProto(TbSubscription subscription) { + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + TbSubscriptionProto subscriptionProto = TbSubscriptionProto.newBuilder() + .setServiceId(subscription.getServiceId()) + .setSessionId(subscription.getSessionId()) + .setSubscriptionId(subscription.getSubscriptionId()) + .setTenantIdMSB(subscription.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(subscription.getTenantId().getId().getLeastSignificantBits()) + .setEntityType(subscription.getEntityId().getEntityType().name()) + .setEntityIdMSB(subscription.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(subscription.getEntityId().getId().getLeastSignificantBits()).build(); + + switch (subscription.getType()) { + case TIMESERIES: + TbTimeseriesSubscription tSub = (TbTimeseriesSubscription) subscription; + TbTimeSeriesSubscriptionProto.Builder tSubProto = TbTimeSeriesSubscriptionProto.newBuilder() + .setSub(subscriptionProto) + .setAllKeys(tSub.isAllKeys()); + tSub.getKeyStates().forEach((key, value) -> tSubProto.addKeyStates( + TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); + tSubProto.setStartTime(tSub.getStartTime()); + tSubProto.setEndTime(tSub.getEndTime()); + msgBuilder.setTelemetrySub(tSubProto.build()); + break; + case ATTRIBUTES: + TbAttributeSubscription aSub = (TbAttributeSubscription) subscription; + TbAttributeSubscriptionProto.Builder aSubProto = TbAttributeSubscriptionProto.newBuilder() + .setSub(subscriptionProto) + .setAllKeys(aSub.isAllKeys()) + .setScope(aSub.getScope().name()); + aSub.getKeyStates().forEach((key, value) -> aSubProto.addKeyStates( + TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); + msgBuilder.setAttributeSub(aSubProto.build()); + break; + } + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static ToCoreMsg toCloseSubscriptionProto(TbSubscription subscription) { + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + TbSubscriptionCloseProto closeProto = TbSubscriptionCloseProto.newBuilder() + .setSessionId(subscription.getSessionId()) + .setSubscriptionId(subscription.getSubscriptionId()).build(); + msgBuilder.setSubClose(closeProto); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static TbSubscription fromProto(TbAttributeSubscriptionProto attributeSub) { + TbSubscriptionProto subProto = attributeSub.getSub(); + TbAttributeSubscription.TbAttributeSubscriptionBuilder builder = TbAttributeSubscription.builder() + .serviceId(subProto.getServiceId()) + .sessionId(subProto.getSessionId()) + .subscriptionId(subProto.getSubscriptionId()) + .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) + .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); + + builder.scope(TbAttributeSubscriptionScope.valueOf(attributeSub.getScope())); + builder.allKeys(attributeSub.getAllKeys()); + Map keyStates = new HashMap<>(); + attributeSub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); + builder.keyStates(keyStates); + return builder.build(); + } + + public static TbSubscription fromProto(TbTimeSeriesSubscriptionProto telemetrySub) { + TbSubscriptionProto subProto = telemetrySub.getSub(); + TbTimeseriesSubscription.TbTimeseriesSubscriptionBuilder builder = TbTimeseriesSubscription.builder() + .serviceId(subProto.getServiceId()) + .sessionId(subProto.getSessionId()) + .subscriptionId(subProto.getSubscriptionId()) + .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) + .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); + + builder.allKeys(telemetrySub.getAllKeys()); + Map keyStates = new HashMap<>(); + telemetrySub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); + builder.startTime(telemetrySub.getStartTime()); + builder.endTime(telemetrySub.getEndTime()); + builder.keyStates(keyStates); + return builder.build(); + } + + public static SubscriptionUpdate fromProto(TbSubscriptionUpdateProto proto) { + if (proto.getErrorCode() > 0) { + return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); + } else { + Map> data = new TreeMap<>(); + proto.getDataList().forEach(v -> { + List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); + for (int i = 0; i < v.getTsCount(); i++) { + Object[] value = new Object[2]; + value[0] = v.getTs(i); + value[1] = v.getValue(i); + values.add(value); + } + }); + return new SubscriptionUpdate(proto.getSubscriptionId(), data); + } + } + + public static ToCoreMsg toTimeseriesUpdateProto(TenantId tenantId, EntityId entityId, List ts) { + TbTimeSeriesUpdateProto.Builder builder = TbTimeSeriesUpdateProto.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + msgBuilder.setTsUpdate(builder); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static ToCoreMsg toAttributesUpdateProto(TenantId tenantId, EntityId entityId, String scope, List attributes) { + TbAttributeUpdateProto.Builder builder = TbAttributeUpdateProto.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setScope(scope); + attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); + + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + msgBuilder.setAttrUpdate(builder); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + private static TsKvProto.Builder toKeyValueProto(long ts, KvEntry attr) { + KeyValueProto.Builder dataBuilder = KeyValueProto.newBuilder(); + dataBuilder.setKey(attr.getKey()); + dataBuilder.setType(KeyValueType.forNumber(attr.getDataType().ordinal())); + switch (attr.getDataType()) { + case BOOLEAN: + attr.getBooleanValue().ifPresent(dataBuilder::setBoolV); + break; + case LONG: + attr.getLongValue().ifPresent(dataBuilder::setLongV); + break; + case DOUBLE: + attr.getDoubleValue().ifPresent(dataBuilder::setDoubleV); + break; + case JSON: + attr.getJsonValue().ifPresent(dataBuilder::setJsonV); + break; + case STRING: + attr.getStrValue().ifPresent(dataBuilder::setStringV); + break; + } + return TsKvProto.newBuilder().setTs(ts).setKv(dataBuilder); + } + + public static EntityId toEntityId(String entityType, long entityIdMSB, long entityIdLSB) { + return EntityIdFactory.getByTypeAndUuid(entityType, new UUID(entityIdMSB, entityIdLSB)); + } + + public static List toTsKvEntityList(List dataList) { + List result = new ArrayList<>(dataList.size()); + dataList.forEach(proto -> result.add(new BasicTsKvEntry(proto.getTs(), getKvEntry(proto.getKv())))); + return result; + } + + public static List toAttributeKvList(List dataList) { + List result = new ArrayList<>(dataList.size()); + dataList.forEach(proto -> result.add(new BaseAttributeKvEntry(getKvEntry(proto.getKv()), proto.getTs()))); + return result; + } + + private static KvEntry getKvEntry(KeyValueProto proto) { + KvEntry entry = null; + DataType type = DataType.values()[proto.getType().getNumber()]; + switch (type) { + case BOOLEAN: + entry = new BooleanDataEntry(proto.getKey(), proto.getBoolV()); + break; + case LONG: + entry = new LongDataEntry(proto.getKey(), proto.getLongV()); + break; + case DOUBLE: + entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleV()); + break; + case STRING: + entry = new StringDataEntry(proto.getKey(), proto.getStringV()); + break; + case JSON: + entry = new JsonDataEntry(proto.getKey(), proto.getJsonV()); + break; + } + return entry; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java new file mode 100644 index 0000000000..0be63f7b65 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.Builder; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +public class TbTimeseriesSubscription extends TbSubscription { + + @Getter private final boolean allKeys; + @Getter private final Map keyStates; + @Getter private final long startTime; + @Getter private final long endTime; + + @Builder + public TbTimeseriesSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, + boolean allKeys, Map keyStates, long startTime, long endTime) { + super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.TIMESERIES); + this.allKeys = allKeys; + this.keyStates = keyStates; + this.startTime = startTime; + this.endTime = endTime; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index d29e5de5b7..665a787e6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -18,73 +18,43 @@ package org.thingsboard.server.service.telemetry; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; -import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; -import org.thingsboard.server.service.state.DefaultDeviceStateService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.sub.Subscription; -import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.subscription.SubscriptionManagerService; +import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.TreeMap; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -93,39 +63,36 @@ import java.util.stream.Collectors; @Slf4j public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptionService { - @Autowired - private TelemetryWebSocketService wsService; + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); - @Autowired - private AttributesService attrService; + private final AttributesService attrService; + private final TimeseriesService tsService; + private final TbClusterService clusterService; + private final PartitionService partitionService; + private Optional subscriptionManagerService; - @Autowired - private TimeseriesService tsService; - - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService rpcService; - - @Autowired - private EntityViewService entityViewService; - - @Autowired - @Lazy - private DeviceStateService stateService; - - @Autowired - @Lazy - private ActorService actorService; - private ExecutorService tsCallBackExecutor; private ExecutorService wsCallBackExecutor; + public DefaultTelemetrySubscriptionService(AttributesService attrService, + TimeseriesService tsService, + TbClusterService clusterService, + PartitionService partitionService) { + this.attrService = attrService; + this.tsService = tsService; + this.clusterService = clusterService; + this.partitionService = partitionService; + } + + @Autowired(required = false) + public void setSubscriptionManagerService(Optional subscriptionManagerService) { + this.subscriptionManagerService = subscriptionManagerService; + } + @PostConstruct public void initExecutor() { - tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); - wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); + tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); + wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ws-callback")); } @PreDestroy @@ -138,64 +105,12 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } } - private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); - private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); - @Override - public void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub) { - long startTime = 0L; - long endTime = 0L; - if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TelemetryFeature.TIMESERIES.equals(sub.getType())) { - EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(entityId.getId())); - entityId = entityView.getEntityId(); - startTime = entityView.getStartTimeMs(); - endTime = entityView.getEndTimeMs(); - sub = getUpdatedSubscriptionState(entityId, sub, entityView); - } - Optional server = routingService.resolveById(entityId); - Subscription subscription; - if (server.isPresent()) { - ServerAddress address = server.get(); - log.trace("[{}] Forwarding subscription [{}] for [{}] entity [{}] to [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId, address); - subscription = new Subscription(sub, true, address, startTime, endTime); - tellNewSubscription(address, sessionId, subscription); - } else { - log.trace("[{}] Registering local subscription [{}] for [{}] entity [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId); - subscription = new Subscription(sub, true, null, startTime, endTime); - } - registerSubscription(sessionId, entityId, subscription); - } - - private SubscriptionState getUpdatedSubscriptionState(EntityId entityId, SubscriptionState sub, EntityView entityView) { - Map keyStates; - if (sub.isAllKeys()) { - keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); - } else { - keyStates = sub.getKeyStates().entrySet() - .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - return new SubscriptionState(sub.getWsSessionId(), sub.getSubscriptionId(), sub.getTenantId(), entityId, sub.getType(), false, keyStates, sub.getScope()); - } - - @Override - public void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId) { - cleanupLocalWsSessionSubscriptions(sessionId); - } - - @Override - public void removeSubscription(String sessionId, int subscriptionId) { - log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); - Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); - if (sessionSubscriptions != null) { - Subscription subscription = sessionSubscriptions.remove(subscriptionId); - if (subscription != null) { - processSubscriptionRemoval(sessionId, sessionSubscriptions, subscription); - } else { - log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); - } - } else { - log.debug("[{}] No session subscriptions found!", sessionId); + @EventListener(PartitionChangeEvent.class) + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); } } @@ -208,14 +123,14 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onTimeseriesUpdate(entityId, ts)); + addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, FutureCallback callback) { ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onAttributesUpdate(entityId, scope, attributes)); + addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes)); } @Override @@ -242,352 +157,31 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio , System.currentTimeMillis())), callback); } - @Override - public void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes) { - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate(tenantId, - deviceId, DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); - } - - @Override - public void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionProto proto; - try { - proto = ClusterAPIProtos.SubscriptionProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - Map statesMap = proto.getKeyStatesList().stream().collect( - Collectors.toMap(ClusterAPIProtos.SubscriptionKetStateProto::getKey, ClusterAPIProtos.SubscriptionKetStateProto::getTs)); - Subscription subscription = new Subscription( - new SubscriptionState(proto.getSessionId(), proto.getSubscriptionId(), - new TenantId(UUID.fromString(proto.getTenantId())), - EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), - TelemetryFeature.valueOf(proto.getType()), proto.getAllKeys(), statesMap, proto.getScope()), - false, new ServerAddress(serverAddress.getHost(), serverAddress.getPort(), serverAddress.getServerType())); - - addRemoteWsSubscription(serverAddress, proto.getSessionId(), subscription); - } - - @Override - public void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionUpdateProto proto; - try { - proto = ClusterAPIProtos.SubscriptionUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - SubscriptionUpdate update = convert(proto); - String sessionId = proto.getSessionId(); - log.trace("[{}] Processing remote subscription onUpdate [{}]", sessionId, update); - Optional subOpt = getSubscription(sessionId, update.getSubscriptionId()); - if (subOpt.isPresent()) { - updateSubscriptionState(sessionId, subOpt.get(), update); - wsService.sendWsMsg(sessionId, update); - } - } - - @Override - public void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionCloseProto proto; - try { - proto = ClusterAPIProtos.SubscriptionCloseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - removeSubscription(proto.getSessionId(), proto.getSubscriptionId()); - } - - @Override - public void onRemoteSessionClose(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SessionCloseProto proto; - try { - proto = ClusterAPIProtos.SessionCloseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - cleanupRemoteWsSessionSubscriptions(proto.getSessionId()); - } - - @Override - public void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.AttributeUpdateProto proto; - try { - proto = ClusterAPIProtos.AttributeUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - onAttributesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), proto.getScope(), - proto.getDataList().stream().map(this::toAttribute).collect(Collectors.toList())); - } - - @Override - public void onRemoteTsUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.TimeseriesUpdateProto proto; - try { - proto = ClusterAPIProtos.TimeseriesUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - onTimeseriesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), - proto.getDataList().stream().map(this::toTimeseries).collect(Collectors.toList())); - } - - @Override - public void onClusterUpdate() { - log.trace("Processing cluster onUpdate msg!"); - Iterator>> deviceIterator = subscriptionsByEntityId.entrySet().iterator(); - while (deviceIterator.hasNext()) { - Map.Entry> e = deviceIterator.next(); - Set subscriptions = e.getValue(); - Optional newAddressOptional = routingService.resolveById(e.getKey()); - if (newAddressOptional.isPresent()) { - newAddressOptional.ifPresent(serverAddress -> checkSubscriptionsNewAddress(serverAddress, subscriptions)); + private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + if (currentPartitions.contains(tpi)) { + if (subscriptionManagerService.isPresent()) { + subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbCallback.EMPTY); } else { - checkSubscriptionsPrevAddress(subscriptions); - } - if (subscriptions.size() == 0) { - log.trace("[{}] No more subscriptions for this device on current server.", e.getKey()); - deviceIterator.remove(); - } - } - } - - private void checkSubscriptionsNewAddress(ServerAddress newAddress, Set subscriptions) { - Iterator subscriptionIterator = subscriptions.iterator(); - while (subscriptionIterator.hasNext()) { - Subscription s = subscriptionIterator.next(); - if (s.isLocal()) { - if (!newAddress.equals(s.getServer())) { - log.trace("[{}] Local subscription is now handled on new server [{}]", s.getWsSessionId(), newAddress); - s.setServer(newAddress); - tellNewSubscription(newAddress, s.getWsSessionId(), s); - } - } else { - log.trace("[{}] Remote subscription is now handled on new server address: [{}]", s.getWsSessionId(), newAddress); - subscriptionIterator.remove(); - //TODO: onUpdate state of subscription by WsSessionId and other maps. - } - } - } - - private void checkSubscriptionsPrevAddress(Set subscriptions) { - for (Subscription s : subscriptions) { - if (s.isLocal() && s.getServer() != null) { - log.trace("[{}] Local subscription is no longer handled on remote server address [{}]", s.getWsSessionId(), s.getServer()); - s.setServer(null); - } else { - log.trace("[{}] Remote subscription is on up to date server address.", s.getWsSessionId()); - } - } - } - - private void addRemoteWsSubscription(ServerAddress address, String sessionId, Subscription subscription) { - EntityId entityId = subscription.getEntityId(); - log.trace("[{}] Registering remote subscription [{}] for entity [{}] to [{}]", sessionId, subscription.getSubscriptionId(), entityId, address); - registerSubscription(sessionId, entityId, subscription); - if (subscription.getType() == TelemetryFeature.ATTRIBUTES) { - final Map keyStates = subscription.getKeyStates(); - DonAsynchron.withCallback(attrService.find(subscription.getSub().getTenantId(), entityId, DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { - List missedUpdates = new ArrayList<>(); - values.forEach(latestEntry -> { - if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { - missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); - } - }); - if (!missedUpdates.isEmpty()) { - tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); - } - }, - e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); - } else if (subscription.getType() == TelemetryFeature.TIMESERIES) { - long curTs = System.currentTimeMillis(); - List queries = new ArrayList<>(); - subscription.getKeyStates().entrySet().forEach(e -> { - if (curTs > e.getValue()) { - queries.add(new BaseReadTsKvQuery(e.getKey(), e.getValue() + 1L, curTs, 0, 1000, Aggregation.NONE)); - } else { - log.debug("[{}] Invalid subscription [{}], entityId [{}] curTs [{}]", sessionId, subscription, entityId, curTs); - } - }); - if (!queries.isEmpty()) { - DonAsynchron.withCallback(tsService.findAll(subscription.getSub().getTenantId(), entityId, queries), - missedUpdates -> { - if (missedUpdates != null && !missedUpdates.isEmpty()) { - tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); - } - }, - e -> log.error("Failed to fetch missed updates.", e), - tsCallBackExecutor); - } - } - } - - private void onAttributesUpdate(EntityId entityId, String scope, List attributes) { - Optional serverAddress = routingService.resolveById(entityId); - if (!serverAddress.isPresent()) { - onLocalAttributesUpdate(entityId, scope, attributes); - if (entityId.getEntityType() == EntityType.DEVICE && DataConstants.SERVER_SCOPE.equalsIgnoreCase(scope)) { - for (AttributeKvEntry attribute : attributes) { - if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { - stateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); - } - } + log.warn("Possible misconfiguration because subscriptionManagerService is null!"); } } else { - tellRemoteAttributesUpdate(serverAddress.get(), entityId, scope, attributes); - } - } - - private void onTimeseriesUpdate(EntityId entityId, List ts) { - Optional serverAddress = routingService.resolveById(entityId); - if (!serverAddress.isPresent()) { - onLocalTimeseriesUpdate(entityId, ts); - } else { - tellRemoteTimeseriesUpdate(serverAddress.get(), entityId, ts); - } - } - - private void onLocalAttributesUpdate(EntityId entityId, String scope, List attributes) { - onLocalSubUpdate(entityId, s -> TelemetryFeature.ATTRIBUTES == s.getType() && (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope())), s -> { - List subscriptionUpdate = null; - for (AttributeKvEntry kv : attributes) { - if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); - } - } - return subscriptionUpdate; - }); - } - - private void onLocalTimeseriesUpdate(EntityId entityId, List ts) { - onLocalSubUpdate(entityId, s -> TelemetryFeature.TIMESERIES == s.getType(), s -> { - List subscriptionUpdate = null; - for (TsKvEntry kv : ts) { - if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(kv); - } - } - return subscriptionUpdate; - }); - } - - private boolean isInTimeRange(Subscription subscription, long kvTime) { - return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) - && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); - } - - private void onLocalSubUpdate(EntityId entityId, Predicate filter, Function> f) { - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - if (deviceSubscriptions != null) { - deviceSubscriptions.stream().filter(filter).forEach(s -> { - String sessionId = s.getWsSessionId(); - List subscriptionUpdate = f.apply(s); - if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { - SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); - if (s.isLocal()) { - updateSubscriptionState(sessionId, s, update); - wsService.sendWsMsg(sessionId, update); - } else { - tellRemoteSubUpdate(s.getServer(), sessionId, update); - } - } - }); - } else { - log.debug("[{}] No device subscriptions to process!", entityId); - } - } - - private void updateSubscriptionState(String sessionId, Subscription subState, SubscriptionUpdate update) { - log.trace("[{}] updating subscription state {} using onUpdate {}", sessionId, subState, update); - update.getLatestValues().entrySet().forEach(e -> subState.setKeyState(e.getKey(), e.getValue())); - } - - private void registerSubscription(String sessionId, EntityId entityId, Subscription subscription) { - Set deviceSubscriptions = subscriptionsByEntityId.computeIfAbsent(entityId, k -> ConcurrentHashMap.newKeySet()); - deviceSubscriptions.add(subscription); - Map sessionSubscriptions = subscriptionsByWsSessionId.computeIfAbsent(sessionId, k -> new ConcurrentHashMap<>()); - sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); - } - - private void cleanupLocalWsSessionSubscriptions(String sessionId) { - cleanupWsSessionSubscriptions(sessionId, true); - } - - private void cleanupRemoteWsSessionSubscriptions(String sessionId) { - cleanupWsSessionSubscriptions(sessionId, false); - } - - private void cleanupWsSessionSubscriptions(String sessionId, boolean localSession) { - log.debug("[{}] Removing all subscriptions for particular session.", sessionId); - Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); - if (sessionSubscriptions != null) { - int sessionSubscriptionSize = sessionSubscriptions.size(); - - for (Subscription subscription : sessionSubscriptions.values()) { - EntityId entityId = subscription.getEntityId(); - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - deviceSubscriptions.remove(subscription); - if (deviceSubscriptions.isEmpty()) { - subscriptionsByEntityId.remove(entityId); - } - } - subscriptionsByWsSessionId.remove(sessionId); - log.debug("[{}] Removed {} subscriptions for particular session.", sessionId, sessionSubscriptionSize); - - if (localSession) { - notifyWsSubscriptionClosed(sessionId, sessionSubscriptions); - } - } else { - log.debug("[{}] No subscriptions found!", sessionId); - } - } - - private void notifyWsSubscriptionClosed(String sessionId, Map sessionSubscriptions) { - Set affectedServers = new HashSet<>(); - for (Subscription subscription : sessionSubscriptions.values()) { - if (subscription.getServer() != null) { - affectedServers.add(subscription.getServer()); - } - } - for (ServerAddress address : affectedServers) { - log.debug("[{}] Going to onSubscriptionUpdate [{}] server about session close event", sessionId, address); - tellRemoteSessionClose(address, sessionId); + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); + clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); } } - private void processSubscriptionRemoval(String sessionId, Map sessionSubscriptions, Subscription subscription) { - EntityId entityId = subscription.getEntityId(); - if (subscription.isLocal() && subscription.getServer() != null) { - tellRemoteSubClose(subscription.getServer(), sessionId, subscription.getSubscriptionId()); - } - if (sessionSubscriptions.isEmpty()) { - log.debug("[{}] Removed last subscription for particular session.", sessionId); - subscriptionsByWsSessionId.remove(sessionId); - } else { - log.debug("[{}] Removed session subscription.", sessionId); - } - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - if (deviceSubscriptions != null) { - boolean result = deviceSubscriptions.remove(subscription); - if (result) { - if (deviceSubscriptions.size() == 0) { - log.debug("[{}] Removed last subscription for particular device.", sessionId); - subscriptionsByEntityId.remove(entityId); - } else { - log.debug("[{}] Removed device subscription.", sessionId); - } + private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + if (currentPartitions.contains(tpi)) { + if (subscriptionManagerService.isPresent()) { + subscriptionManagerService.get().onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY); } else { - log.debug("[{}] Subscription not found!", sessionId); + log.warn("Possible misconfiguration because subscriptionManagerService is null!"); } } else { - log.debug("[{}] No device subscriptions found!", sessionId); + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); + clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); } } @@ -617,161 +211,4 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } }, wsCallBackExecutor); } - - private void tellNewSubscription(ServerAddress address, String sessionId, Subscription sub) { - ClusterAPIProtos.SubscriptionProto.Builder builder = ClusterAPIProtos.SubscriptionProto.newBuilder(); - builder.setSessionId(sessionId); - builder.setSubscriptionId(sub.getSubscriptionId()); - builder.setTenantId(sub.getSub().getTenantId().getId().toString()); - builder.setEntityType(sub.getEntityId().getEntityType().name()); - builder.setEntityId(sub.getEntityId().getId().toString()); - builder.setType(sub.getType().name()); - builder.setAllKeys(sub.isAllKeys()); - if (sub.getScope() != null) { - builder.setScope(sub.getScope()); - } - sub.getKeyStates().entrySet().forEach(e -> builder.addKeyStates( - ClusterAPIProtos.SubscriptionKetStateProto.newBuilder().setKey(e.getKey()).setTs(e.getValue()).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteSubUpdate(ServerAddress address, String sessionId, SubscriptionUpdate update) { - ClusterAPIProtos.SubscriptionUpdateProto.Builder builder = ClusterAPIProtos.SubscriptionUpdateProto.newBuilder(); - builder.setSessionId(sessionId); - builder.setSubscriptionId(update.getSubscriptionId()); - builder.setErrorCode(update.getErrorCode()); - if (update.getErrorMsg() != null) { - builder.setErrorMsg(update.getErrorMsg()); - } - update.getData().entrySet().forEach( - e -> { - ClusterAPIProtos.SubscriptionUpdateValueListProto.Builder dataBuilder = ClusterAPIProtos.SubscriptionUpdateValueListProto.newBuilder(); - - dataBuilder.setKey(e.getKey()); - e.getValue().forEach(v -> { - Object[] array = (Object[]) v; - dataBuilder.addTs((long) array[0]); - dataBuilder.addValue((String) array[1]); - }); - - builder.addData(dataBuilder.build()); - } - ); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteAttributesUpdate(ServerAddress address, EntityId entityId, String scope, List attributes) { - ClusterAPIProtos.AttributeUpdateProto.Builder builder = ClusterAPIProtos.AttributeUpdateProto.newBuilder(); - builder.setEntityId(entityId.getId().toString()); - builder.setEntityType(entityId.getEntityType().name()); - builder.setScope(scope); - attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteTimeseriesUpdate(ServerAddress address, EntityId entityId, List ts) { - ClusterAPIProtos.TimeseriesUpdateProto.Builder builder = ClusterAPIProtos.TimeseriesUpdateProto.newBuilder(); - builder.setEntityId(entityId.getId().toString()); - builder.setEntityType(entityId.getEntityType().name()); - ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteSessionClose(ServerAddress address, String sessionId) { - ClusterAPIProtos.SessionCloseProto proto = ClusterAPIProtos.SessionCloseProto.newBuilder().setSessionId(sessionId).build(); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); - } - - private void tellRemoteSubClose(ServerAddress address, String sessionId, int subscriptionId) { - ClusterAPIProtos.SubscriptionCloseProto proto = ClusterAPIProtos.SubscriptionCloseProto.newBuilder().setSessionId(sessionId).setSubscriptionId(subscriptionId).build(); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); - } - - private ClusterAPIProtos.KeyValueProto.Builder toKeyValueProto(long ts, KvEntry attr) { - ClusterAPIProtos.KeyValueProto.Builder dataBuilder = ClusterAPIProtos.KeyValueProto.newBuilder(); - dataBuilder.setKey(attr.getKey()); - dataBuilder.setTs(ts); - dataBuilder.setValueType(attr.getDataType().ordinal()); - switch (attr.getDataType()) { - case BOOLEAN: - Optional booleanValue = attr.getBooleanValue(); - booleanValue.ifPresent(dataBuilder::setBoolValue); - break; - case LONG: - Optional longValue = attr.getLongValue(); - longValue.ifPresent(dataBuilder::setLongValue); - break; - case DOUBLE: - Optional doubleValue = attr.getDoubleValue(); - doubleValue.ifPresent(dataBuilder::setDoubleValue); - break; - case JSON: - Optional jsonValue = attr.getJsonValue(); - jsonValue.ifPresent(dataBuilder::setJsonValue); - break; - case STRING: - Optional stringValue = attr.getStrValue(); - stringValue.ifPresent(dataBuilder::setStrValue); - break; - } - return dataBuilder; - } - - private AttributeKvEntry toAttribute(ClusterAPIProtos.KeyValueProto proto) { - return new BaseAttributeKvEntry(getKvEntry(proto), proto.getTs()); - } - - private TsKvEntry toTimeseries(ClusterAPIProtos.KeyValueProto proto) { - return new BasicTsKvEntry(proto.getTs(), getKvEntry(proto)); - } - - private KvEntry getKvEntry(ClusterAPIProtos.KeyValueProto proto) { - KvEntry entry = null; - DataType type = DataType.values()[proto.getValueType()]; - switch (type) { - case BOOLEAN: - entry = new BooleanDataEntry(proto.getKey(), proto.getBoolValue()); - break; - case LONG: - entry = new LongDataEntry(proto.getKey(), proto.getLongValue()); - break; - case DOUBLE: - entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleValue()); - break; - case STRING: - entry = new StringDataEntry(proto.getKey(), proto.getStrValue()); - break; - case JSON: - entry = new JsonDataEntry(proto.getKey(), proto.getJsonValue()); - break; - } - return entry; - } - - private SubscriptionUpdate convert(ClusterAPIProtos.SubscriptionUpdateProto proto) { - if (proto.getErrorCode() > 0) { - return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); - } else { - Map> data = new TreeMap<>(); - proto.getDataList().forEach(v -> { - List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); - for (int i = 0; i < v.getTsCount(); i++) { - Object[] value = new Object[2]; - value[0] = v.getTs(i); - value[1] = v.getValue(i); - values.add(value); - } - }); - return new SubscriptionUpdate(proto.getSubscriptionId(), data); - } - } - - private Optional getSubscription(String sessionId, int subscriptionId) { - Subscription state = null; - Map subMap = subscriptionsByWsSessionId.get(sessionId); - if (subMap != null) { - state = subMap.get(subscriptionId); - } - return Optional.ofNullable(state); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 741efe6f8a..5e9e83b8aa 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -43,12 +43,18 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.TenantRateLimitException; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.ValidationCallback; import org.thingsboard.server.service.security.ValidationResult; import org.thingsboard.server.service.security.ValidationResultCode; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; +import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; +import org.thingsboard.server.service.subscription.TbAttributeSubscription; +import org.thingsboard.server.service.subscription.TbTimeseriesSubscription; import org.thingsboard.server.service.telemetry.cmd.AttributesSubscriptionCmd; import org.thingsboard.server.service.telemetry.cmd.GetHistoryCmd; import org.thingsboard.server.service.telemetry.cmd.SubscriptionCmd; @@ -57,7 +63,6 @@ import org.thingsboard.server.service.telemetry.cmd.TelemetryPluginCmdsWrapper; import org.thingsboard.server.service.telemetry.cmd.TimeseriesSubscriptionCmd; import org.thingsboard.server.service.telemetry.exception.UnauthorizedException; import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; import javax.annotation.Nullable; @@ -83,6 +88,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 27.03.18. */ @Service +@TbCoreComponent @Slf4j public class DefaultTelemetryWebSocketService implements TelemetryWebSocketService { @@ -98,7 +104,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private final ConcurrentMap wsSessionsMap = new ConcurrentHashMap<>(); @Autowired - private TelemetrySubscriptionService subscriptionManager; + private TbLocalSubscriptionService subService; @Autowired private TelemetryWebSocketMsgEndpoint msgEndpoint; @@ -112,6 +118,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Autowired private TimeseriesService tsService; + @Autowired + private TbServiceInfoProvider serviceInfoProvider; + @Value("${server.ws.limits.max_subscriptions_per_tenant:0}") private int maxSubscriptionsPerTenant; @Value("${server.ws.limits.max_subscriptions_per_customer:0}") @@ -127,9 +136,11 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private ConcurrentMap> publicUserSubscriptionsMap = new ConcurrentHashMap<>(); private ExecutorService executor; + private String serviceId; @PostConstruct public void initExecutor() { + serviceId = serviceInfoProvider.getServiceId(); executor = Executors.newWorkStealingPool(50); } @@ -153,7 +164,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi break; case CLOSED: wsSessionsMap.remove(sessionId); - subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); + subService.cancelAllSessionSubscriptions(sessionId); processSessionClose(sessionRef); break; } @@ -334,8 +345,16 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi keys.forEach(key -> subState.put(key, 0L)); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, false, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + TbAttributeSubscription sub = TbAttributeSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(false) + .keyStates(subState) + .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + subService.addSubscription(sub); } @Override @@ -421,8 +440,16 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi Map subState = new HashMap<>(attributesData.size()); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, true, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + TbAttributeSubscription sub = TbAttributeSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(true) + .keyStates(subState) + .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + subService.addSubscription(sub); } @Override @@ -494,8 +521,16 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); Map subState = new HashMap<>(data.size()); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, true, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + + TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(true) + .keyStates(subState).build(); + subService.addSubscription(sub); } @Override @@ -520,12 +555,19 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Override public void onSuccess(List data) { sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); - Map subState = new HashMap<>(keys.size()); keys.forEach(key -> subState.put(key, startTs)); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, false, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + + TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(false) + .keyStates(subState).build(); + subService.addSubscription(sub); } @Override @@ -544,9 +586,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private void unsubscribe(TelemetryWebSocketSessionRef sessionRef, SubscriptionCmd cmd, String sessionId) { if (cmd.getEntityId() == null || cmd.getEntityId().isEmpty()) { - subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); + subService.cancelAllSessionSubscriptions(sessionId); } else { - subscriptionManager.removeSubscription(sessionId, cmd.getCmdId()); + subService.cancelSubscription(sessionId, cmd.getCmdId()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index d77a0e50dc..d652a3f1c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -15,34 +15,13 @@ */ package org.thingsboard.server.service.telemetry; +import org.springframework.context.ApplicationListener; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; /** * Created by ashvayka on 27.03.18. */ -public interface TelemetrySubscriptionService extends RuleEngineTelemetryService { +public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener { - void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub); - - void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId); - - void removeSubscription(String sessionId, int cmdId); - - void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data); - - void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] bytes); - - void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] bytes); - - void onRemoteSessionClose(ServerAddress serverAddress, byte[] bytes); - - void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] bytes); - - void onRemoteTsUpdate(ServerAddress serverAddress, byte[] bytes); - - void onClusterUpdate(); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java deleted file mode 100644 index 7f7db9430d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.telemetry.sub; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.service.telemetry.TelemetryFeature; - -import java.util.Map; - -@Data -@AllArgsConstructor -public class Subscription { - - private final SubscriptionState sub; - private final boolean local; - private ServerAddress server; - private long startTime; - private long endTime; - - public Subscription(SubscriptionState sub, boolean local, ServerAddress server) { - this(sub, local, server, 0L, 0L); - } - - public String getWsSessionId() { - return getSub().getWsSessionId(); - } - - public int getSubscriptionId() { - return getSub().getSubscriptionId(); - } - - public EntityId getEntityId() { - return getSub().getEntityId(); - } - - public TelemetryFeature getType() { - return getSub().getType(); - } - - public String getScope() { - return getSub().getScope(); - } - - public boolean isAllKeys() { - return getSub().isAllKeys(); - } - - public Map getKeyStates() { - return getSub().getKeyStates(); - } - - public void setKeyState(String key, long ts) { - getSub().getKeyStates().put(key, ts); - } - - @Override - public String toString() { - return "Subscription{" + - "sub=" + sub + - ", local=" + local + - ", server=" + server + - '}'; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java index 68aaca34c2..992dc26af9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java @@ -75,7 +75,7 @@ public class SubscriptionUpdate { if (data == null) { return Collections.emptyMap(); } else { - return data.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> { + return data.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> { List data = e.getValue(); Object[] latest = (Object[]) data.get(data.size() - 1); return (long) latest[0]; diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java deleted file mode 100644 index 1aee532c93..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ /dev/null @@ -1,242 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transaction; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; -import org.thingsboard.server.service.executors.DbCallbackExecutorService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Optional; -import java.util.Queue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; - -@Service -@Slf4j -public class BaseRuleChainTransactionService implements RuleChainTransactionService { - - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService clusterRpcService; - - @Autowired - private DbCallbackExecutorService callbackExecutor; - - @Value("${actors.rule.transaction.queue_size}") - private int finalQueueSize; - @Value("${actors.rule.transaction.duration}") - private long duration; - - private final Lock transactionLock = new ReentrantLock(); - private final ConcurrentMap> transactionMap = new ConcurrentHashMap<>(); - private final Queue timeoutQueue = new ConcurrentLinkedQueue<>(); - - private ExecutorService timeoutExecutor; - - @PostConstruct - public void init() { - timeoutExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("rule-chain-transaction")); - executeOnTimeout(); - } - - @PreDestroy - public void destroy() { - if (timeoutExecutor != null) { - timeoutExecutor.shutdownNow(); - } - } - - @Override - public void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure) { - transactionLock.lock(); - try { - BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> - new LinkedBlockingQueue<>(finalQueueSize)); - - TbTransactionTask transactionTask = new TbTransactionTask(msg, onStart, onEnd, onFailure, System.currentTimeMillis() + duration); - int queueSize = queue.size(); - if (queueSize >= finalQueueSize) { - log.trace("Queue has no space: {}", transactionTask); - executeOnFailure(transactionTask.getOnFailure(), "Queue has no space!"); - } else { - addMsgToQueues(queue, transactionTask); - if (queueSize == 0) { - executeOnSuccess(transactionTask.getOnStart(), transactionTask.getMsg()); - } else { - log.trace("Msg [{}][{}] is waiting to start transaction!", msg.getId(), msg.getType()); - } - } - } finally { - transactionLock.unlock(); - } - } - - @Override - public void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { - Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); - if (address.isPresent()) { - sendTransactionEventToRemoteServer(msg, address.get()); - executeOnSuccess(onSuccess, msg); - } else { - endLocalTransaction(msg, onSuccess, onFailure); - } - } - - @Override - public void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] data) { - endLocalTransaction(TbMsg.fromBytes(data), msg -> { - }, error -> { - }); - } - - private void addMsgToQueues(BlockingQueue queue, TbTransactionTask transactionTask) { - queue.offer(transactionTask); - timeoutQueue.offer(transactionTask); - log.trace("Added msg to queue, size: [{}]", queue.size()); - } - - private void endLocalTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { - transactionLock.lock(); - try { - BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> - new LinkedBlockingQueue<>(finalQueueSize)); - - TbTransactionTask currentTransactionTask = queue.peek(); - if (currentTransactionTask != null) { - if (currentTransactionTask.getMsg().getTransactionData().getTransactionId().equals(msg.getTransactionData().getTransactionId())) { - currentTransactionTask.setCompleted(true); - queue.poll(); - log.trace("Removed msg from queue, size [{}]", queue.size()); - - executeOnSuccess(currentTransactionTask.getOnEnd(), currentTransactionTask.getMsg()); - executeOnSuccess(onSuccess, msg); - - TbTransactionTask nextTransactionTask = queue.peek(); - if (nextTransactionTask != null) { - executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); - } - } else { - log.trace("Task has expired!"); - executeOnFailure(onFailure, "Task has expired!"); - } - } else { - log.trace("Queue is empty, previous task has expired!"); - executeOnFailure(onFailure, "Queue is empty, previous task has expired!"); - } - } finally { - transactionLock.unlock(); - } - } - - private void executeOnTimeout() { - timeoutExecutor.submit(() -> { - while (true) { - TbTransactionTask transactionTask = timeoutQueue.peek(); - if (transactionTask != null) { - long sleepDuration = 0L; - transactionLock.lock(); - try { - if (transactionTask.isCompleted()) { - timeoutQueue.poll(); - } else { - long expIn = transactionTask.getExpirationTime() - System.currentTimeMillis(); - if (expIn < 0) { - log.trace("Task has expired! Deleting it...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); - timeoutQueue.poll(); - executeOnFailure(transactionTask.getOnFailure(), "Task has expired!"); - - BlockingQueue queue = transactionMap.get(transactionTask.getMsg().getTransactionData().getOriginatorId()); - if (queue != null) { - queue.poll(); - TbTransactionTask nextTransactionTask = queue.peek(); - if (nextTransactionTask != null) { - executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); - } - } - } else { - sleepDuration = Math.min(expIn, duration); - } - } - } finally { - transactionLock.unlock(); - } - if (sleepDuration > 0L) { - try { - log.trace("Task has not expired! Continue executing...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); - TimeUnit.MILLISECONDS.sleep(sleepDuration); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted", e); - } - } - } else { - try { - log.trace("Queue is empty, waiting for tasks!"); - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted", e); - } - } - } - }); - } - - private void executeOnFailure(Consumer onFailure, String exception) { - executeCallback(() -> { - onFailure.accept(new RuntimeException(exception)); - return null; - }); - } - - private void executeOnSuccess(Consumer onSuccess, TbMsg tbMsg) { - executeCallback(() -> { - onSuccess.accept(tbMsg); - return null; - }); - } - - private void executeCallback(Callable task) { - callbackExecutor.executeAsync(task); - } - - private void sendTransactionEventToRemoteServer(TbMsg msg, ServerAddress address) { - log.trace("[{}][{}] Originator is monitored on other server: {}", msg.getTransactionData().getOriginatorId(), msg.getTransactionData().getTransactionId(), address); - clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java b/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java deleted file mode 100644 index c86f882ff7..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transaction; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.common.msg.TbMsg; - -import java.util.function.Consumer; - -@Data -@AllArgsConstructor -public final class TbTransactionTask { - - private final TbMsg msg; - private final Consumer onStart; - private final Consumer onEnd; - private final Consumer onFailure; - private final long expirationTime; - - private boolean isCompleted; - - public TbTransactionTask(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure, long expirationTime) { - this.msg = msg; - this.onStart = onStart; - this.onEnd = onEnd; - this.onFailure = onFailure; - this.expirationTime = expirationTime; - this.isCompleted = false; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java new file mode 100644 index 0000000000..c9e37e6791 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -0,0 +1,87 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.UUID; +import java.util.function.Consumer; + +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; + +@Slf4j +@Service +@TbCoreComponent +public class DefaultTbCoreToTransportService implements TbCoreToTransportService { + + private final PartitionService partitionService; + private final TbQueueProducer> tbTransportProducer; + + public DefaultTbCoreToTransportService(PartitionService partitionService, TbQueueProducerProvider tbQueueProducerProvider) { + this.partitionService = partitionService; + this.tbTransportProducer = tbQueueProducerProvider.getTransportNotificationsMsgProducer(); + } + + @Override + public void process(String nodeId, ToTransportMsg msg) { + process(nodeId, msg, null, null); + } + + @Override + public void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, nodeId); + UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); + log.trace("[{}][{}] Pushing session data to topic: {}", tpi.getFullTopicName(), sessionId, msg); + TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, msg); + tbTransportProducer.send(tpi, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + } + + private static class QueueCallbackAdaptor implements TbQueueCallback { + private final Runnable onSuccess; + private final Consumer onFailure; + + QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (onSuccess != null) { + onSuccess.run(); + } + } + + @Override + public void onFailure(Throwable t) { + if (onFailure != null) { + onFailure.accept(t); + } + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java rename to application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 8e820fbffe..7456ca7662 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -25,6 +25,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -33,14 +34,19 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; @@ -52,10 +58,15 @@ import java.util.concurrent.locks.ReentrantLock; */ @Slf4j @Service -public class LocalTransportApiService implements TransportApiService { +@TbCoreComponent +public class DefaultTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); + //TODO: Constructor dependencies; + @Autowired + private TenantService tenantService; + @Autowired private DeviceService deviceService; @@ -74,17 +85,20 @@ public class LocalTransportApiService implements TransportApiService { private ReentrantLock deviceCreationLock = new ReentrantLock(); @Override - public ListenableFuture handle(TransportApiRequestMsg transportApiRequestMsg) { + public ListenableFuture> handle(TbProtoQueueMsg tbProtoQueueMsg) { + TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue(); if (transportApiRequestMsg.hasValidateTokenRequestMsg()) { ValidateDeviceTokenRequestMsg msg = transportApiRequestMsg.getValidateTokenRequestMsg(); - return validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN); + return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } else if (transportApiRequestMsg.hasValidateX509CertRequestMsg()) { ValidateDeviceX509CertRequestMsg msg = transportApiRequestMsg.getValidateX509CertRequestMsg(); - return validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE); + return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) { - return handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()); + return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + } else if (transportApiRequestMsg.hasGetTenantRoutingInfoRequestMsg()) { + return Futures.transform(handle(transportApiRequestMsg.getGetTenantRoutingInfoRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } - return getEmptyTransportApiResponseFuture(); + return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } private ListenableFuture validateCredentials(String credentialsId, DeviceCredentialsType credentialsType) { @@ -125,6 +139,13 @@ public class LocalTransportApiService implements TransportApiService { }, dbCallbackExecutorService); } + private ListenableFuture handle(GetTenantRoutingInfoRequestMsg requestMsg) { + TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB())); + ListenableFuture tenantFuture = tenantService.findTenantByIdAsync(TenantId.SYS_TENANT_ID, tenantId); + return Futures.transform(tenantFuture, tenant -> TransportApiResponseMsg.newBuilder() + .setGetTenantRoutingInfoResponseMsg(GetTenantRoutingInfoResponseMsg.newBuilder().setIsolatedTbCore(tenant.isIsolatedTbCore()) + .setIsolatedTbRuleEngine(tenant.isIsolatedTbRuleEngine()).build()).build(), dbCallbackExecutorService); + } private ListenableFuture getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { return Futures.transform(deviceService.findDeviceByIdAsync(TenantId.SYS_TENANT_ID, deviceId), device -> { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java deleted file mode 100644 index 32ff77877d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java +++ /dev/null @@ -1,227 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.DonAsynchron; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.service.AbstractTransportService; -import org.thingsboard.server.dao.device.ClaimDevicesService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Optional; -import java.util.UUID; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 12.10.18. - */ -@Slf4j -@Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "local") -public class LocalTransportService extends AbstractTransportService implements RuleEngineTransportService { - - @Autowired - private TransportApiService transportApiService; - - @Autowired - private ActorSystemContext actorContext; - - //TODO: completely replace this routing with the Kafka routing by partition ids. - @Autowired - private ClusterRoutingService routingService; - @Autowired - private ClusterRpcService rpcService; - @Autowired - private DataDecodingEncodingService encodingService; - @Autowired - private ClaimDevicesService claimDevicesService; - - @PostConstruct - public void init() { - super.init(); - } - - @PreDestroy - public void destroy() { - super.destroy(); - } - - @Override - public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getGetOrCreateDeviceResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSessionEvent(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostTelemetry(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostAttributes(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setGetAttributes(msg).build(), callback); - } - - @Override - public void process(SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscriptionInfo(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToAttributes(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToRPC(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToDeviceRPCCallResponse(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToServerRPCCallRequest(msg).build(), callback); - } - - @Override - protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { - TransportToDeviceActorMsg toDeviceActorMsg = TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setClaimDevice(msg).build(); - - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - callback.onSuccess(null); - } else { - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(msg.getDeviceIdMSB(), msg.getDeviceIdLSB())); - DonAsynchron.withCallback(claimDevicesService.registerClaimingInfo(tenantId, deviceId, msg.getSecretKey(), msg.getDurationMs()), - callback::onSuccess, callback::onError); - } - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg) { - process(nodeId, msg, null, null); - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - processToTransportMsg(msg); - if (onSuccess != null) { - onSuccess.run(); - } - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - } else { - actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); - } - if (callback != null) { - callback.onSuccess(null); - } - } - - private Consumer getThrowableConsumer(TransportServiceCallback callback) { - return e -> { - if (callback != null) { - callback.onError(e); - } - }; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java deleted file mode 100644 index 7aa438c64f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java +++ /dev/null @@ -1,251 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import akka.actor.ActorRef; -import io.github.bucket4j.Bandwidth; -import io.github.bucket4j.BlockingBucket; -import io.github.bucket4j.Bucket4j; -import io.github.bucket4j.local.LocalBucket; -import io.github.bucket4j.local.LocalBucketBuilder; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.time.Duration; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 09.10.18. - */ -@Slf4j -@Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") -public class RemoteRuleEngineTransportService implements RuleEngineTransportService { - - @Value("${transport.remote.rule_engine.topic}") - private String ruleEngineTopic; - @Value("${transport.remote.notifications.topic}") - private String notificationsTopic; - @Value("${transport.remote.rule_engine.poll_interval}") - private int pollDuration; - @Value("${transport.remote.rule_engine.auto_commit_interval}") - private int autoCommitInterval; - - @Value("${transport.remote.rule_engine.poll_records_pack_size}") - private int pollRecordsPackSize; - @Value("${transport.remote.rule_engine.max_poll_records_per_second}") - private long pollRecordsPerSecond; - @Value("${transport.remote.rule_engine.max_poll_records_per_minute}") - private long pollRecordsPerMinute; - @Value("${transport.remote.rule_engine.stats.enabled:false}") - private boolean statsEnabled; - - @Autowired - private TbKafkaSettings kafkaSettings; - - @Autowired - private TbNodeIdProvider nodeIdProvider; - - @Autowired - private ActorSystemContext actorContext; - - //TODO: completely replace this routing with the Kafka routing by partition ids. - @Autowired - private ClusterRoutingService routingService; - @Autowired - private ClusterRpcService rpcService; - @Autowired - private DataDecodingEncodingService encodingService; - - private TBKafkaConsumerTemplate ruleEngineConsumer; - private TBKafkaProducerTemplate notificationsProducer; - - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-main-consumer")); - - private volatile boolean stopped = false; - - private final RuleEngineStats stats = new RuleEngineStats(); - - @PostConstruct - public void init() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder notificationsProducerBuilder = TBKafkaProducerTemplate.builder(); - notificationsProducerBuilder.settings(kafkaSettings); - notificationsProducerBuilder.clientId("producer-transport-notification-" + nodeIdProvider.getNodeId()); - notificationsProducerBuilder.encoder(new ToTransportMsgEncoder()); - - notificationsProducer = notificationsProducerBuilder.build(); - notificationsProducer.init(); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder ruleEngineConsumerBuilder = TBKafkaConsumerTemplate.builder(); - ruleEngineConsumerBuilder.settings(kafkaSettings); - ruleEngineConsumerBuilder.topic(ruleEngineTopic); - ruleEngineConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); - ruleEngineConsumerBuilder.groupId("tb-node"); - ruleEngineConsumerBuilder.autoCommit(true); - ruleEngineConsumerBuilder.autoCommitIntervalMs(autoCommitInterval); - ruleEngineConsumerBuilder.maxPollRecords(pollRecordsPackSize); - ruleEngineConsumerBuilder.decoder(new ToRuleEngineMsgDecoder()); - - ruleEngineConsumer = ruleEngineConsumerBuilder.build(); - ruleEngineConsumer.subscribe(); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting polling for events."); - LocalBucketBuilder builder = Bucket4j.builder(); - builder.addLimit(Bandwidth.simple(pollRecordsPerSecond, Duration.ofSeconds(1))); - builder.addLimit(Bandwidth.simple(pollRecordsPerMinute, Duration.ofMinutes(1))); - LocalBucket pollRateBucket = builder.build(); - BlockingBucket blockingPollRateBucket = pollRateBucket.asScheduler(); - - mainConsumerExecutor.execute(() -> { - while (!stopped) { - try { - ConsumerRecords records = ruleEngineConsumer.poll(Duration.ofMillis(pollDuration)); - int recordsCount = records.count(); - if (recordsCount > 0) { - while (!blockingPollRateBucket.tryConsume(recordsCount, TimeUnit.SECONDS.toNanos(5))) { - log.info("Rule Engine consumer is busy. Required tokens: [{}]. Available tokens: [{}].", recordsCount, pollRateBucket.getAvailableTokens()); - Thread.sleep(TimeUnit.SECONDS.toMillis(1)); - } - log.trace("Processing {} records", recordsCount); - } - records.forEach(record -> { - try { - ToRuleEngineMsg toRuleEngineMsg = ruleEngineConsumer.decode(record); - log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); - if (toRuleEngineMsg.hasToDeviceActorMsg()) { - forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg()); - } - } catch (Throwable e) { - log.warn("Failed to process the notification.", e); - } - }); - } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - @Scheduled(fixedDelayString = "${transport.remote.rule_engine.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - stats.printStats(); - } - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg) { - process(nodeId, msg, null, null); - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - String topic = notificationsTopic + "." + nodeId; - UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); - ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); - log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); - notificationsProducer.send(topic, sessionId.toString(), transportMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - log.trace("[{}] Pushing message to remote server: {}", address.get(), toDeviceActorMsg); - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - } else { - log.trace("Pushing message to local server: {}", toDeviceActorMsg); - actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); - } - } - - @PreDestroy - public void destroy() { - stopped = true; - if (ruleEngineConsumer != null) { - ruleEngineConsumer.unsubscribe(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - } - - private static class QueueCallbackAdaptor implements Callback { - private final Runnable onSuccess; - private final Consumer onFailure; - - QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { - this.onSuccess = onSuccess; - this.onFailure = onFailure; - } - - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception == null) { - if (onSuccess != null) { - onSuccess.run(); - } - } else { - if (onFailure != null) { - onFailure.accept(exception); - } - } - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java deleted file mode 100644 index 0c9efc7d1c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.*; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.concurrent.*; - -/** - * Created by ashvayka on 05.10.18. - */ -@Slf4j -@Component -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") -public class RemoteTransportApiService { - - @Value("${transport.remote.transport_api.requests_topic}") - private String transportApiRequestsTopic; - @Value("${transport.remote.transport_api.max_pending_requests}") - private int maxPendingRequests; - @Value("${transport.remote.transport_api.request_timeout}") - private long requestTimeout; - @Value("${transport.remote.transport_api.request_poll_interval}") - private int responsePollDuration; - @Value("${transport.remote.transport_api.request_auto_commit_interval}") - private int autoCommitInterval; - - @Autowired - private TbKafkaSettings kafkaSettings; - - @Autowired - private TbNodeIdProvider nodeIdProvider; - - @Autowired - private TransportApiService transportApiService; - - private ExecutorService transportCallbackExecutor; - - private TbKafkaResponseTemplate transportApiTemplate; - - @PostConstruct - public void init() { - this.transportCallbackExecutor = Executors.newWorkStealingPool(100); - - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder responseBuilder = TBKafkaProducerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.clientId("producer-transport-api-response-" + nodeIdProvider.getNodeId()); - responseBuilder.encoder(new TransportApiResponseEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder requestBuilder = TBKafkaConsumerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.topic(transportApiRequestsTopic); - requestBuilder.clientId(nodeIdProvider.getNodeId()); - requestBuilder.groupId("tb-node"); - requestBuilder.autoCommit(true); - requestBuilder.autoCommitIntervalMs(autoCommitInterval); - requestBuilder.decoder(new TransportApiRequestDecoder()); - - TbKafkaResponseTemplate.TbKafkaResponseTemplateBuilder - builder = TbKafkaResponseTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(maxPendingRequests); - builder.requestTimeout(requestTimeout); - builder.pollInterval(responsePollDuration); - builder.executor(transportCallbackExecutor); - builder.handler(transportApiService); - transportApiTemplate = builder.build(); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting polling for events."); - transportApiTemplate.init(); - } - - @PreDestroy - public void destroy() { - if (transportApiTemplate != null) { - transportApiTemplate.stop(); - } - if (transportCallbackExecutor != null) { - transportCallbackExecutor.shutdownNow(); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java rename to application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java index 9c3f5634bf..e0d5e2121a 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java @@ -16,14 +16,13 @@ package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; -/** - * Created by ashvayka on 05.10.18. - */ -public class ToTransportMsgEncoder implements TbKafkaEncoder { - @Override - public byte[] encode(ToTransportMsg value) { - return value.toByteArray(); - } +import java.util.function.Consumer; + +public interface TbCoreToTransportService { + + void process(String nodeId, ToTransportMsg msg); + + void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure); + } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java new file mode 100644 index 0000000000..566bf99cd3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -0,0 +1,100 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.transport; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.concurrent.*; + +/** + * Created by ashvayka on 05.10.18. + */ +@Slf4j +@Service +@TbCoreComponent +public class TbCoreTransportApiService { + + private final TbCoreQueueFactory tbCoreQueueFactory; + private final TransportApiService transportApiService; + + @Value("${queue.transport_api.max_pending_requests:10000}") + private int maxPendingRequests; + @Value("${queue.transport_api.max_requests_timeout:10000}") + private long requestTimeout; + @Value("${queue.transport_api.request_poll_interval:25}") + private int responsePollDuration; + @Value("${queue.transport_api.max_callback_threads:100}") + private int maxCallbackThreads; + + private ExecutorService transportCallbackExecutor; + private TbQueueResponseTemplate, + TbProtoQueueMsg> transportApiTemplate; + + public TbCoreTransportApiService(TbCoreQueueFactory tbCoreQueueFactory, TransportApiService transportApiService) { + this.tbCoreQueueFactory = tbCoreQueueFactory; + this.transportApiService = transportApiService; + } + + @PostConstruct + public void init() { + this.transportCallbackExecutor = Executors.newWorkStealingPool(maxCallbackThreads); + TbQueueProducer> producer = tbCoreQueueFactory.createTransportApiResponseProducer(); + TbQueueConsumer> consumer = tbCoreQueueFactory.createTransportApiRequestConsumer(); + + DefaultTbQueueResponseTemplate.DefaultTbQueueResponseTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueResponseTemplate.builder(); + builder.requestTemplate(consumer); + builder.responseTemplate(producer); + builder.maxPendingRequests(maxPendingRequests); + builder.requestTimeout(requestTimeout); + builder.pollInterval(responsePollDuration); + builder.executor(transportCallbackExecutor); + builder.handler(transportApiService); + transportApiTemplate = builder.build(); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + log.info("Received application ready event. Starting polling for events."); + transportApiTemplate.init(transportApiService); + } + + @PreDestroy + public void destroy() { + if (transportApiTemplate != null) { + transportApiTemplate.stop(); + } + if (transportCallbackExecutor != null) { + transportCallbackExecutor.shutdownNow(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java deleted file mode 100644 index ac1c5965f3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; - -import java.io.IOException; - -/** - * Created by ashvayka on 05.10.18. - */ -public class TransportApiRequestDecoder implements TbKafkaDecoder { - @Override - public TransportApiRequestMsg decode(byte[] data) throws IOException { - return TransportApiRequestMsg.parseFrom(data); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java deleted file mode 100644 index 35f8a7b1aa..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.kafka.TbKafkaEncoder; - -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; - -/** - * Created by ashvayka on 05.10.18. - */ -public class TransportApiResponseEncoder implements TbKafkaEncoder { - @Override - public byte[] encode(TransportApiResponseMsg value) { - return value.toByteArray(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java index 2964934313..c9edfa9331 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.service.transport; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.kafka.TbKafkaHandler; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; /** * Created by ashvayka on 05.10.18. */ -public interface TransportApiService extends TbKafkaHandler { +public interface TransportApiService extends TbQueueHandler, TbProtoQueueMsg> { } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java index 34abac72bb..ba4e7baab1 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; import java.io.Serializable; import java.util.UUID; @@ -36,9 +37,11 @@ public class TransportToDeviceActorMsgWrapper implements TbActorMsg, DeviceAware private final TenantId tenantId; private final DeviceId deviceId; private final TransportToDeviceActorMsg msg; + private final TbCallback callback; - public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg) { + public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg, TbCallback callback) { this.msg = msg; + this.callback = callback; this.tenantId = new TenantId(new UUID(msg.getSessionInfo().getTenantIdMSB(), msg.getSessionInfo().getTenantIdLSB())); this.deviceId = new DeviceId(new UUID(msg.getSessionInfo().getDeviceIdMSB(), msg.getSessionInfo().getDeviceIdLSB())); } diff --git a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java index 1b8173c424..4b7f0f6366 100644 --- a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.UpdateMessage; +import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -38,6 +39,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @Service +@TbCoreComponent @Slf4j public class DefaultUpdateService implements UpdateService { diff --git a/application/src/main/proto/cluster.proto b/application/src/main/proto/cluster.proto deleted file mode 100644 index cfacc66121..0000000000 --- a/application/src/main/proto/cluster.proto +++ /dev/null @@ -1,146 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; -package cluster; - -option java_package = "org.thingsboard.server.gen.cluster"; -option java_outer_classname = "ClusterAPIProtos"; - -service ClusterRpcService { - rpc handleMsgs(stream ClusterMessage) returns (stream ClusterMessage) {} -} - -message ClusterMessage { - MessageType messageType = 1; - MessageMataInfo messageMetaInfo = 2; - ServerAddress serverAddress = 3; - bytes payload = 4; -} - -message ServerAddress { - string host = 1; - int32 port = 2; -} - -message MessageMataInfo { - string payloadMetaInfo = 1; - repeated string tags = 2; -} - -enum MessageType { - - //Cluster control messages - RPC_SESSION_CREATE_REQUEST_MSG = 0; - TO_ALL_NODES_MSG = 1; - RPC_SESSION_TELL_MSG = 2; - RPC_BROADCAST_MSG = 3; - CONNECT_RPC_MESSAGE =4; - - CLUSTER_ACTOR_MESSAGE = 5; - // Messages related to TelemetrySubscriptionService - CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE = 6; - CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE = 7; - CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE = 8; - CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE = 9; - CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE = 10; - CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE = 11; - CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE = 12; - - CLUSTER_DEVICE_STATE_SERVICE_MESSAGE = 13; - CLUSTER_TRANSACTION_SERVICE_MESSAGE = 14; -} - -// Messages related to CLUSTER_TELEMETRY_MESSAGE -message SubscriptionProto { - string sessionId = 1; - int32 subscriptionId = 2; - string entityType = 3; - string tenantId = 4; - string entityId = 5; - string type = 6; - bool allKeys = 7; - repeated SubscriptionKetStateProto keyStates = 8; - string scope = 9; -} - -message SubscriptionUpdateProto { - string sessionId = 1; - int32 subscriptionId = 2; - int32 errorCode = 3; - string errorMsg = 4; - repeated SubscriptionUpdateValueListProto data = 5; -} - -message AttributeUpdateProto { - string entityType = 1; - string entityId = 2; - string scope = 3; - repeated KeyValueProto data = 4; -} - -message TimeseriesUpdateProto { - string entityType = 1; - string entityId = 2; - repeated KeyValueProto data = 4; -} - -message SessionCloseProto { - string sessionId = 1; -} - -message SubscriptionCloseProto { - string sessionId = 1; - int32 subscriptionId = 2; -} - -message SubscriptionKetStateProto { - string key = 1; - int64 ts = 2; -} - -message SubscriptionUpdateValueListProto { - string key = 1; - repeated int64 ts = 2; - repeated string value = 3; -} - -message KeyValueProto { - string key = 1; - int64 ts = 2; - int32 valueType = 3; - string strValue = 4; - int64 longValue = 5; - double doubleValue = 6; - bool boolValue = 7; - string jsonValue = 8; -} - -message FromDeviceRPCResponseProto { - int64 requestIdMSB = 1; - int64 requestIdLSB = 2; - string response = 3; - int32 error = 4; -} - -message DeviceStateServiceMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 deviceIdMSB = 3; - int64 deviceIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index e147325f42..614f85ba0e 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -28,6 +28,9 @@ + + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0402c91387..3df3cb9559 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -31,7 +31,7 @@ server: key-store-type: "${SSL_KEY_STORE_TYPE:PKCS12}" # Alias that identifies the key in the key store key-alias: "${SSL_KEY_ALIAS:tomcat}" - log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:true}" + log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:false}" ws: send_timeout: "${TB_SERVER_WS_SEND_TIMEOUT:5000}" limits: @@ -70,21 +70,7 @@ zk: # Name of the directory in zookeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" -# RPC connection parameters. Used only in cluster mode only. -rpc: - bind_host: "${RPC_HOST:localhost}" - bind_port: "${RPC_PORT:9001}" - -# Clustering properties related to consistent-hashing. See architecture docs for more details. cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" - # Name of hash function used for consistent hash ring. - hash_function_name: "${CLUSTER_HASH_FUNCTION_NAME:murmur3_128}" - # Amount of virtual nodes in consistent hash ring. - vitrual_nodes_size: "${CLUSTER_VIRTUAL_NODES_SIZE:16}" - # Queue partition id for current node - partition_id: "${QUEUE_PARTITION_ID:0}" stats: enabled: "${TB_CLUSTER_STATS_ENABLED:false}" print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" @@ -195,36 +181,34 @@ cassandra: # SQL configuration parameters sql: - # Specify batch size for persisting attribute updates - attributes: - batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" - ts: - batch_size: "${SQL_TS_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" - ts_latest: - batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}" - # Specify whether to remove null characters from strValue of attributes and timeseries before insert - remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" - postgres: - # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE. - ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}" - timescale: - # Specify Interval size for new data chunks storage. - chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" - ttl: - enabled: "${SQL_TTL_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds - ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds + # Specify batch size for persisting attribute updates + attributes: + batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" + batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" + stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" + ts: + batch_size: "${SQL_TS_BATCH_SIZE:10000}" + batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}" + stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" + ts_latest: + batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" + batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" + stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}" + # Specify whether to remove null characters from strValue of attributes and timeseries before insert + remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" + postgres: + # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE. + ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}" + timescale: + # Specify Interval size for new data chunks storage. + chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" + ttl: + enabled: "${SQL_TTL_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds + ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds # Actor system parameters actors: - cluster: - grpc_callback_thread_pool_size: "${ACTORS_CLUSTER_GRPC_CALLBACK_THREAD_POOL_SIZE:10}" tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" session: @@ -267,8 +251,6 @@ actors: enabled: "${ACTORS_QUEUE_ENABLED:true}" # Maximum allowed timeout for persistence into the queue timeout: "${ACTORS_QUEUE_PERSISTENCE_TIMEOUT:30000}" - client_side_rpc: - timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" cache: # caffeine or redis @@ -348,19 +330,19 @@ updates: # spring CORS configuration spring.mvc.cors: - mappings: - # Intercept path - "[/api/**]": - #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. - allowed-origins: "*" - #Comma-separated list of methods to allow. '*' allows all methods. - allowed-methods: "*" - #Comma-separated list of headers to allow in a request. '*' allows all headers. - allowed-headers: "*" - #How long, in seconds, the response from a pre-flight request can be cached by clients. - max-age: "1800" - #Set whether credentials are supported. When not set, credentials are not supported. - allow-credentials: "true" + mappings: + # Intercept path + "[/api/**]": + #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. + allowed-origins: "*" + #Comma-separated list of methods to allow. '*' allows all methods. + allowed-methods: "*" + #Comma-separated list of headers to allow in a request. '*' allows all headers. + allowed-headers: "*" + #How long, in seconds, the response from a pre-flight request can be cached by clients. + max-age: "1800" + #Set whether credentials are supported. When not set, credentials are not supported. + allow-credentials: "true" # spring serve gzip compressed static resources spring.resources.chain: @@ -388,7 +370,7 @@ spring: username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: - maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:50}" + maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:5}" # Audit log parameters audit-log: @@ -432,28 +414,6 @@ state: defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - request_poll_interval: "${TB_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - request_auto_commit_interval: "${TB_TRANSPORT_REQUEST_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - js: evaluator: "${JS_EVALUATOR:local}" # local/remote # Built-in JVM JavaScript environment properties @@ -463,55 +423,27 @@ js: # Specify thread pool size for JavaScript sandbox resource monitor monitor_thread_pool_size: "${LOCAL_JS_SANDBOX_MONITOR_THREAD_POOL_SIZE:4}" # Maximum CPU time in milliseconds allowed for script execution - max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:3000}" + max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:10000}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${LOCAL_JS_SANDBOX_MAX_ERRORS:3}" # JS Eval max request timeout. 0 - no timeout max_requests_timeout: "${LOCAL_JS_MAX_REQUEST_TIMEOUT:0}" + # Maximum time in seconds for black listed function to stay in the list. + max_black_list_duration_sec: "${LOCAL_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_LOCAL_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_LOCAL_STATS_PRINT_INTERVAL_MS:10000}" # Remote JavaScript environment properties remote: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${REMOTE_JS_SANDBOX_MAX_ERRORS:3}" + # Maximum time in seconds for black listed function to stay in the list. + max_black_list_duration_sec: "${REMOTE_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_REMOTE_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_REMOTE_STATS_PRINT_INTERVAL_MS:10000}" transport: - type: "${TRANSPORT_TYPE:local}" # local or remote - remote: - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - request_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - request_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - request_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:1000}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" - poll_records_pack_size: "${TB_RULE_ENGINE_MAX_POLL_RECORDS:1000}" - max_poll_records_per_second: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_SECOND:10000}" - max_poll_records_per_minute: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_MINUTE:120000}" - stats: - enabled: "${TB_RULE_ENGINE_STATS_ENABLED:false}" - print_interval_ms: "${TB_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" @@ -524,6 +456,8 @@ transport: type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" + client_side_rpc: + timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" # Local HTTP transport parameters http: enabled: "${HTTP_ENABLED:true}" @@ -567,7 +501,7 @@ swagger: api_path_regex: "${SWAGGER_API_PATH_REGEX:/api.*}" security_path_regex: "${SWAGGER_SECURITY_PATH_REGEX:/api.*}" non_security_path_regex: "${SWAGGER_NON_SECURITY_PATH_REGEX:/api/noauth.*}" - title: "${SWAGGER_TITLE:Thingsboard REST API}" + title: "${SWAGGER_TITLE:ThingsBoard REST API}" description: "${SWAGGER_DESCRIPTION:For instructions how to authorize requests please visit REST API documentation page.}" contact: name: "${SWAGGER_CONTACT_NAME:Thingsboard team}" @@ -577,3 +511,145 @@ swagger: title: "${SWAGGER_LICENSE_TITLE:Apache License Version 2.0}" url: "${SWAGGER_LICENSE_URL:https://github.com/thingsboard/thingsboard/blob/master/LICENSE}" version: "${SWAGGER_VERSION:2.0}" + +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" + transport_api: + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. + diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index 50e3580405..26f375c464 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -33,6 +33,7 @@ import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.junit.runner.RunWith; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootContextLoader; import org.springframework.boot.test.context.SpringBootTest; @@ -114,9 +115,7 @@ public abstract class AbstractControllerTest { */ private static final long DEFAULT_TIMEOUT = -1L; - protected MediaType contentType = new MediaType(MediaType.APPLICATION_JSON.getType(), - MediaType.APPLICATION_JSON.getSubtype(), - Charset.forName("utf8")); + protected MediaType contentType = MediaType.APPLICATION_JSON; protected MockMvc mockMvc; @@ -199,6 +198,7 @@ public abstract class AbstractControllerTest { createUserAndLogin(customerUser, CUSTOMER_USER_PASSWORD); logout(); + log.info("Executed setup"); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java index 56c73b5e13..0422a85416 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; import java.util.ArrayList; import java.util.Collections; @@ -71,7 +72,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) + doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) .andExpect(status().isOk()); } @@ -111,26 +112,27 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { @Test public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Asset asset = new Asset(); - asset.setName("My asset B"+i); + asset.setName("My asset B" + i); asset.setType("typeB"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Asset asset = new Asset(); - asset.setName("My asset C"+i); + asset.setName("My asset C" + i); asset.setType("typeC"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Asset asset = new Asset(); - asset.setName("My asset A"+i); + asset.setName("My asset A" + i); asset.setType("typeA"); assets.add(doPost("/api/asset", asset, Asset.class)); } List assetTypes = doGetTyped("/api/asset/types", - new TypeReference>(){}); + new TypeReference>() { + }); Assert.assertNotNull(assetTypes); Assert.assertEquals(3, assetTypes.size()); @@ -146,10 +148,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { asset.setType("default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); - doDelete("/api/asset/"+savedAsset.getId().getId().toString()) + doDelete("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isOk()); - doGet("/api/asset/"+savedAsset.getId().getId().toString()) + doGet("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isNotFound()); } @@ -244,16 +246,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant2.getId().getId().toString()) + doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) .andExpect(status().isOk()); } @Test public void testFindTenantAssets() throws Exception { List assets = new ArrayList<>(); - for (int i=0;i<178;i++) { + for (int i = 0; i < 178; i++) { Asset asset = new Asset(); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); assets.add(doPost("/api/asset", asset, Asset.class)); } @@ -262,13 +264,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssets.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); } } while (pageData.hasNext()); + loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE)); + Collections.sort(assets, idComparator); Collections.sort(loadedAssets, idComparator); @@ -279,10 +284,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void testFindTenantAssetsByName() throws Exception { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -290,10 +295,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -305,7 +310,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -321,7 +327,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -334,24 +341,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsTitle2, loadedAssetsTitle2); for (Asset asset : loadedAssetsTitle1) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4, title1); pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsTitle2) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4, title2); pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -361,10 +370,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -373,10 +382,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -388,7 +397,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); loadedAssetsType1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -404,7 +414,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); do { pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); loadedAssetsType2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -417,24 +428,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsType2, loadedAssetsType2); for (Asset asset : loadedAssetsType1) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsType2) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -447,9 +460,9 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i=0;i<128;i++) { + for (int i = 0; i < 128; i++) { Asset asset = new Asset(); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); asset = doPost("/api/asset", asset, Asset.class); assets.add(doPost("/api/customer/" + customerId.getId().toString() @@ -461,7 +474,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssets.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -483,10 +497,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -496,10 +510,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -513,7 +527,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -529,7 +544,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -548,7 +564,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title1); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); @@ -559,7 +576,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -574,10 +592,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -588,10 +606,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -605,7 +623,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); loadedAssetsType1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -621,7 +640,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); loadedAssetsType2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -640,7 +660,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); @@ -651,7 +672,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index 1eb821f06a..420446e04d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.controller; import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.type.TypeReference; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.eclipse.paho.client.mqttv3.MqttConnectOptions; @@ -24,6 +25,7 @@ import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; @@ -46,6 +48,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; @@ -55,6 +58,7 @@ import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; +@Slf4j public abstract class BaseEntityViewControllerTest extends AbstractControllerTest { private IdComparator idComparator; @@ -417,12 +421,22 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - Thread.sleep(3000); - + awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); MqttMessage message = new MqttMessage(); message.setPayload(strKvs.getBytes()); client.publish("v1/devices/me/telemetry", message); Thread.sleep(1000); + client.disconnect(); + } + + private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException { + long start = System.currentTimeMillis(); + while (!client.isConnected()) { + Thread.sleep(100); + if (start + ms < System.currentTimeMillis()) { + throw new RuntimeException("Client is not connected!"); + } + } } private Set getTelemetryKeys(String type, String id) throws Exception { @@ -449,13 +463,13 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - Thread.sleep(3000); + awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); MqttMessage message = new MqttMessage(); message.setPayload((stringKV).getBytes()); client.publish("v1/devices/me/attributes", message); Thread.sleep(1000); - + client.disconnect(); return new HashSet<>(doGetAsync("/api/plugins/telemetry/DEVICE/" + viewDeviceId + "/keys/attributes", List.class)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java index 4ad41824a0..781c483fc5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.controller; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -37,4 +39,9 @@ public class ControllerNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/system-data.cql", false, false), new ClassPathCQLDataSet("cassandra/system-test.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index 8dc0acff57..347eaee7eb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.controller; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -33,4 +35,9 @@ public class ControllerSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java index 9c4030cff4..7360c5c506 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.mqtt; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -36,4 +38,9 @@ public class MqttNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java index 2863589ba1..70de3b27ca 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.mqtt; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -32,4 +34,9 @@ public class MqttSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 5ba4dfcbae..9ddb2c81d9 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.service.security.AccessValidator; import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -55,6 +56,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC private User tenantAdmin; private Long asyncContextTimeoutToUseRpcPlugin; + private static final AtomicInteger atomicInteger = new AtomicInteger(2); + @Before public void beforeTest() throws Exception { @@ -70,7 +73,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); - tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setEmail("tenant" + atomicInteger.getAndIncrement() + "@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); @@ -109,6 +112,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", MqttQoS.AT_MOST_ONCE.value()); + Thread.sleep(2000); + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -128,7 +133,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"24\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -137,7 +142,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttOneWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"25\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + nonExistentDeviceId, setGpioRequest, String.class, @@ -165,7 +170,9 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", 1); client.setCallback(new TestMqttCallback(client, new CountDownLatch(1))); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + Thread.sleep(2000); + + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -183,7 +190,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"27\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -192,7 +199,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttTwoWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"28\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + nonExistentDeviceId, setGpioRequest, String.class, diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index f1580c65c7..f96f207834 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -79,8 +79,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr String deviceId = savedDevice.getId().getId().toString(); - Thread.sleep(1000); - List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); + Thread.sleep(2000); + List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); Set actualKeySet = new HashSet<>(actualKeys); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4"); @@ -88,7 +88,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals(expectedKeySet, actualKeySet); - String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); + String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); Map>> values = doGetAsync(getTelemetryValuesUrl, Map.class); assertEquals("value1", values.get("key1").get(0).get("value")); @@ -104,13 +104,17 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); - client.connect(options).waitForCompletion(3000); CountDownLatch latch = new CountDownLatch(1); TestMqttCallback callback = new TestMqttCallback(client, latch); client.setCallback(callback); + client.connect(options).waitForCompletion(5000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); String payload = "{\"key\":\"value\"}"; - String result = doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); +// TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. +// MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) +// MqttClient <- SUB_ACK <- Transport + Thread.sleep(5000); + doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); latch.await(10, TimeUnit.SECONDS); assertEquals(payload, callback.getPayload()); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); @@ -120,8 +124,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr private final MqttAsyncClient client; private final CountDownLatch latch; - private Integer qoS; - private String payload; + private volatile Integer qoS; + private volatile String payload; String getPayload() { return payload; @@ -138,6 +142,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr @Override public void connectionLost(Throwable throwable) { + log.error("Client connection lost", throwable); } @Override diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java index e44c383452..fbab13147c 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java @@ -16,11 +16,13 @@ package org.thingsboard.server.rules; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -40,4 +42,9 @@ public class RuleEngineNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java index 5f930821f7..83ac5f7703 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.rules; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -33,4 +35,9 @@ public class RuleEngineSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 6bb1f4d99d..87a181deae 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -15,40 +15,37 @@ */ package org.thingsboard.server.rules.flow; +import akka.actor.ActorRef; import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Lists; -import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.rule.RuleChainService; -import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.function.Predicate; import java.util.stream.Collectors; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -63,7 +60,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule protected User tenantAdmin; @Autowired - protected ActorService actorService; + protected ActorSystemContext actorSystem; @Autowired protected AttributesService attributesService; @@ -150,15 +147,12 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), - "CUSTOM", - device.getId(), - new TbMsgMetaData(), - "{}", null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); @@ -265,15 +259,13 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), - "CUSTOM", - device.getId(), - new TbMsgMetaData(), - "{}", null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + + Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 5f806d1ea0..17f17698c9 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -15,15 +15,17 @@ */ package org.thingsboard.server.rules.lifecycle; +import akka.actor.ActorRef; import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -38,13 +40,13 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; -import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -61,7 +63,7 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac protected User tenantAdmin; @Autowired - protected ActorService actorService; + protected ActorSystemContext actorSystem; @Autowired protected AttributesService attributesService; @@ -136,16 +138,13 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), - "CUSTOM", - device.getId(), - new TbMsgMetaData(), - "{}", - null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); + TimePageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java deleted file mode 100644 index d717169048..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import com.datastax.driver.core.utils.UUIDs; -import lombok.extern.slf4j.Slf4j; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.runners.MockitoJUnitRunner; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.util.ReflectionTestUtils; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.UUIDConverter; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@Slf4j -@RunWith(MockitoJUnitRunner.class) -public class ConsistentClusterRoutingServiceTest { - - private ConsistentClusterRoutingService clusterRoutingService; - - private DiscoveryService discoveryService; - - private String hashFunctionName = "murmur3_128"; - private Integer virtualNodesSize = 1024*4; - private ServerAddress currentServer = new ServerAddress(" 100.96.1.0", 9001, ServerType.CORE); - - - @Before - public void setup() throws Exception { - discoveryService = mock(DiscoveryService.class); - clusterRoutingService = new ConsistentClusterRoutingService(); - ReflectionTestUtils.setField(clusterRoutingService, "discoveryService", discoveryService); - ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); - ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); - when(discoveryService.getCurrentServer()).thenReturn(new ServerInstance(currentServer)); - List otherServers = new ArrayList<>(); - for (int i = 1; i < 30; i++) { - otherServers.add(new ServerInstance(new ServerAddress(" 100.96." + i*2 + "." + i, 9001, ServerType.CORE))); - } - when(discoveryService.getOtherServers()).thenReturn(otherServers); - clusterRoutingService.init(); - } - - @Test - public void testDispersionOnMillionDevices() { - List devices = new ArrayList<>(); - for (int i = 0; i < 1000000; i++) { - devices.add(new DeviceId(UUIDs.timeBased())); - } - - testDevicesDispersion(devices); - } - - private void testDevicesDispersion(List devices) { - long start = System.currentTimeMillis(); - Map map = new HashMap<>(); - for (DeviceId deviceId : devices) { - ServerAddress address = clusterRoutingService.resolveById(deviceId).orElse(currentServer); - map.put(address, map.getOrDefault(address, 0) + 1); - } - - List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); - long end = System.currentTimeMillis(); - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + (data.get(data.size() - 1).getValue() - data.get(0).getValue())); - - for (Map.Entry entry : data) { -// System.out.println(entry.getKey().getHost() + ": " + entry.getValue()); - } - - } - -} diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java new file mode 100644 index 0000000000..3f7619ed1e --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cluster.routing; + +import com.datastax.driver.core.utils.UUIDs; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.ConsistentHashPartitionService; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class ConsistentHashParitionServiceTest { + + public static final int ITERATIONS = 1000000; + private ConsistentHashPartitionService clusterRoutingService; + + private TbServiceInfoProvider discoveryService; + private TenantRoutingInfoService routingInfoService; + private ApplicationEventPublisher applicationEventPublisher; + + private String hashFunctionName = "murmur3_128"; + private Integer virtualNodesSize = 16; + + + @Before + public void setup() throws Exception { + discoveryService = mock(TbServiceInfoProvider.class); + applicationEventPublisher = mock(ApplicationEventPublisher.class); + routingInfoService = mock(TenantRoutingInfoService.class); + clusterRoutingService = new ConsistentHashPartitionService(discoveryService, routingInfoService, applicationEventPublisher); + ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); + ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); + ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); + ReflectionTestUtils.setField(clusterRoutingService, "ruleEnginePartitions", 100); + + ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); + ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); + TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() + .setServiceId("100.96.1.1") + .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) + .build(); +// when(discoveryService.getServiceInfo()).thenReturn(currentServer); + List otherServers = new ArrayList<>(); + for (int i = 1; i < 30; i++) { + otherServers.add(TransportProtos.ServiceInfo.newBuilder() + .setServiceId("100.96." + i * 2 + "." + i) + .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) + .build()); + } + clusterRoutingService.init(); + clusterRoutingService.recalculatePartitions(currentServer, otherServers); + } + + @Test + public void testDispersionOnMillionDevices() { + List devices = new ArrayList<>(); + for (int i = 0; i < ITERATIONS; i++) { + devices.add(new DeviceId(UUIDs.timeBased())); + } + testDevicesDispersion(devices); + } + + private void testDevicesDispersion(List devices) { + long start = System.currentTimeMillis(); + Map map = new HashMap<>(); + for (DeviceId deviceId : devices) { + TopicPartitionInfo address = clusterRoutingService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, deviceId); + Integer partition = address.getPartition().get(); + map.put(partition, map.getOrDefault(partition, 0) + 1); + } + + List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); + long end = System.currentTimeMillis(); + double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); + System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff / ITERATIONS) * 100.0) + "%)"); + + for (Map.Entry entry : data) { + System.out.println(entry.getKey() + ": " + entry.getValue()); + } + + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java new file mode 100644 index 0000000000..2de2414f71 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class ProcessingAttemptContextTest { + + @Test + public void testHighConcurrencyCase() throws InterruptedException { + TbRuleEngineSubmitStrategy strategyMock = mock(TbRuleEngineSubmitStrategy.class); + int msgCount = 1000; + int parallelCount = 5; + ExecutorService executorService = Executors.newFixedThreadPool(parallelCount); + try { + ConcurrentMap> messages = new ConcurrentHashMap<>(); + for (int i = 0; i < msgCount; i++) { + messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); + } + when(strategyMock.getPendingMap()).thenReturn(messages); + ProcessingAttemptContext context = new ProcessingAttemptContext(strategyMock); + for (UUID uuid : messages.keySet()) { + for (int i = 0; i < parallelCount; i++) { + executorService.submit(() -> context.onSuccess(uuid)); + } + } + Assert.assertTrue(context.await(10, TimeUnit.SECONDS)); + Mockito.verify(strategyMock, Mockito.times(msgCount)).onSuccess(Mockito.any(UUID.class)); + } finally { + executorService.shutdownNow(); + } + } +} diff --git a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java index 82b3df9736..62881bd5af 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java @@ -24,6 +24,7 @@ import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -62,7 +63,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("70", actual.getMetaData().getValue("temp")); @@ -78,7 +79,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("94", actual.getMetaData().getValue("newAttr")); @@ -94,7 +95,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); @@ -112,7 +113,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); assertFalse(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -126,7 +127,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson); assertTrue(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -147,7 +148,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one"), actual); scriptEngine.destroy(); @@ -169,7 +170,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one", "three"), actual); scriptEngine.destroy(); diff --git a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java index 1065fbf6d3..8a89de5dff 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java +++ b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java @@ -49,4 +49,9 @@ public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } + + @Override + protected long getMaxBlacklistDuration() { + return 100000; + } } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java index 41be7641c6..c4182db3ee 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.system; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -38,4 +40,9 @@ public class SystemNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java index b12d513ce0..6060a2cc2b 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.system; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -35,4 +37,9 @@ public class SystemSqlTestSuite { "sql/drop-all-tables.sql", "sql-test.properties"); + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } + } diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 47dacce343..7943b9d9b1 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -7,7 +7,7 @@ - + diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index e2277497a5..3132aa99c7 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -18,7 +18,7 @@ package org.thingsboard.server.dao.alarm; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java index 96a36e1f65..a41dd2ee21 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.HSQLDialect") public @interface HsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java index 20954b8ac2..bcf1222988 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.entities.type}'=='cassandra'") public @interface NoSqlAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java index 8b642de026..dde0a9a176 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.entities", value = "type", havingValue = "cassandra") public @interface NoSqlDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java index ebd0ef7d28..7b10e0cf03 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "cassandra") public @interface NoSqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java index ef73ec8f90..a888926bd0 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.PostgreSQLDialect") public @interface PsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java index f2a8800032..a215a16b29 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale') " + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") public @interface PsqlTsAnyDao { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java index 0eea3367e1..408f81a845 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.entities", value = "type", havingValue = "sql") public @interface SqlDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java index 9a43c530a2..9be3321988 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale'") public @interface SqlTsAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java index abba0e985b..0b665c5695 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "sql") public @interface SqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java index 20745d790c..0541a47879 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "timescale") public @interface TimescaleDBTsDao { } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java index b1f6a967b6..766d405e25 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java @@ -29,6 +29,8 @@ public class Tenant extends ContactBased implements HasTenantId { private String title; private String region; + private boolean isolatedTbCore; + private boolean isolatedTbRuleEngine; public Tenant() { super(); @@ -72,6 +74,22 @@ public class Tenant extends ContactBased implements HasTenantId { this.region = region; } + public boolean isIsolatedTbCore() { + return isolatedTbCore; + } + + public void setIsolatedTbCore(boolean isolatedTbCore) { + this.isolatedTbCore = isolatedTbCore; + } + + public boolean isIsolatedTbRuleEngine() { + return isolatedTbRuleEngine; + } + + public void setIsolatedTbRuleEngine(boolean isolatedTbRuleEngine) { + this.isolatedTbRuleEngine = isolatedTbRuleEngine; + } + @Override public String getSearchText() { return getTitle(); @@ -84,6 +102,10 @@ public class Tenant extends ContactBased implements HasTenantId { builder.append(title); builder.append(", region="); builder.append(region); + builder.append(", isolatedTbCore="); + builder.append(isolatedTbCore); + builder.append(", isolatedTbRuleEngine="); + builder.append(isolatedTbRuleEngine); builder.append(", additionalInfo="); builder.append(getAdditionalInfo()); builder.append(", country="); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java index a43d95975e..9c23a60c28 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java @@ -23,6 +23,7 @@ import lombok.Data; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java rename to common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java index 532842ae75..691d89cafd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java @@ -13,14 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.alarm; +package org.thingsboard.server.common.data.id; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.UUIDBased; import java.util.UUID; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index c367355cb0..11db1da1a0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -16,7 +16,6 @@ package org.thingsboard.server.common.data.id; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.alarm.AlarmId; import java.util.UUID; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java index 5639f98d01..2993d81154 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java @@ -32,6 +32,10 @@ public class BaseAttributeKvEntry implements AttributeKvEntry { this.lastUpdateTs = lastUpdateTs; } + public BaseAttributeKvEntry(long lastUpdateTs, KvEntry kv) { + this(kv, lastUpdateTs); + } + @Override public long getLastUpdateTs() { return lastUpdateTs; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 04b0cae56b..0345355aa3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.msg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; + /** * Created by ashvayka on 15.03.18. */ @@ -24,17 +27,12 @@ public enum MsgType { /** * ADDED/UPDATED/DELETED events for server nodes. * - * See {@link org.thingsboard.server.common.msg.cluster.ClusterEventMsg} + * See {@link PartitionChangeMsg} */ - CLUSTER_EVENT_MSG, + PARTITION_CHANGE_MSG, APP_INIT_MSG, - /** - * All messages, could be send to cluster - */ - SEND_TO_CLUSTER_MSG, - /** * ADDED/UPDATED/DELETED events for main entities. * @@ -43,11 +41,11 @@ public enum MsgType { COMPONENT_LIFE_CYCLE_MSG, /** - * Misc messages from the REST API/SERVICE layer to the new rule engine. + * Misc messages consumed from the Queue and forwarded to Rule Engine Actor. * - * See {@link org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg} + * See {@link QueueToRuleEngineMsg} */ - SERVICE_TO_RULE_ENGINE_MSG, + QUEUE_TO_RULE_ENGINE_MSG, /** * Message that is sent by RuleChainActor to RuleActor with command to process TbMsg. @@ -91,12 +89,9 @@ public enum MsgType { DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG, - DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG, - /** * Message that is sent from the Device Actor to Rule Engine. Requires acknowledgement */ - DEVICE_ACTOR_TO_RULE_ENGINE_MSG, SESSION_TIMEOUT_MSG, diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 4e7090f934..3edf4e5061 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -15,24 +15,27 @@ */ package org.thingsboard.server.common.msg; +import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; -import java.nio.ByteBuffer; import java.util.UUID; /** * Created by ashvayka on 13.01.18. */ @Data -@AllArgsConstructor +@Builder +@Slf4j public final class TbMsg implements Serializable { private final UUID id; @@ -41,30 +44,61 @@ public final class TbMsg implements Serializable { private final TbMsgMetaData metaData; private final TbMsgDataType dataType; private final String data; - private final TbMsgTransactionData transactionData; - - //The following fields are not persisted to DB, because they can always be recovered from the context; private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; - private final long clusterPartition; + //This field is not serialized because we use queues and there is no need to do it + private final TbMsgCallback callback; + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); + } + + public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { + return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), + data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); + } + + public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), + tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } - public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { + private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, + RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { this.id = id; this.type = type; this.originator = originator; this.metaData = metaData; + this.dataType = dataType; this.data = data; - this.dataType = TbMsgDataType.JSON; - this.transactionData = new TbMsgTransactionData(id, originator); this.ruleChainId = ruleChainId; this.ruleNodeId = ruleNodeId; - this.clusterPartition = clusterPartition; + if (callback != null) { + this.callback = callback; + } else { + log.warn("[{}] Created message with empty callback: {}", originator, type); + this.callback = TbMsgCallback.EMPTY; + } } - public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { - this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, clusterPartition); + public static ByteString toByteString(TbMsg msg) { + return ByteString.copyFrom(toByteArray(msg)); } public static byte[] toByteArray(TbMsg msg) { @@ -89,52 +123,37 @@ public final class TbMsg implements Serializable { builder.setMetaData(MsgProtos.TbMsgMetaDataProto.newBuilder().putAllData(msg.getMetaData().getData()).build()); } - TbMsgTransactionData transactionData = msg.getTransactionData(); - if (transactionData != null) { - MsgProtos.TbMsgTransactionDataProto.Builder transactionBuilder = MsgProtos.TbMsgTransactionDataProto.newBuilder(); - transactionBuilder.setId(transactionData.getTransactionId().toString()); - transactionBuilder.setEntityType(transactionData.getOriginatorId().getEntityType().name()); - transactionBuilder.setEntityIdMSB(transactionData.getOriginatorId().getId().getMostSignificantBits()); - transactionBuilder.setEntityIdLSB(transactionData.getOriginatorId().getId().getLeastSignificantBits()); - builder.setTransactionData(transactionBuilder.build()); - } builder.setDataType(msg.getDataType().ordinal()); builder.setData(msg.getData()); return builder.build().toByteArray(); - } - public static ByteBuffer toBytes(TbMsg msg) { - return ByteBuffer.wrap(toByteArray(msg)); - } - - public static TbMsg fromBytes(byte[] data) { - return fromBytes(ByteBuffer.wrap(data)); - } - - public static TbMsg fromBytes(ByteBuffer buffer) { + public static TbMsg fromBytes(byte[] data, TbMsgCallback callback) { try { - MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(buffer.array()); + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data); TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap()); - EntityId transactionEntityId = EntityIdFactory.getByTypeAndUuid(proto.getTransactionData().getEntityType(), - new UUID(proto.getTransactionData().getEntityIdMSB(), proto.getTransactionData().getEntityIdLSB())); - TbMsgTransactionData transactionData = new TbMsgTransactionData(UUID.fromString(proto.getTransactionData().getId()), transactionEntityId); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - RuleChainId ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); + RuleChainId ruleChainId = null; RuleNodeId ruleNodeId = null; + if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) { + ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); + } if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) { ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); } TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), transactionData, ruleChainId, ruleNodeId, proto.getClusterPartition()); + return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } } - public TbMsg copy(UUID newId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { - return new TbMsg(newId, type, originator, metaData.copy(), dataType, data, transactionData, ruleChainId, ruleNodeId, clusterPartition); + public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { + return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); } + public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); + } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java deleted file mode 100644 index 80674f9bce..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.cluster; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - * @author Andrew Shvayka - */ -@Data -@EqualsAndHashCode -public class ServerAddress implements Comparable, Serializable { - - private final String host; - private final int port; - private final ServerType serverType; - - @Override - public int compareTo(ServerAddress o) { - int result = this.host.compareTo(o.host); - if (result == 0) { - result = this.port - o.port; - } - return result; - } - - @Override - public String toString() { - return '[' + host + ':' + port + ']'; - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index a52c237f87..d05948206c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -33,7 +33,7 @@ import java.util.Optional; * @author Andrew Shvayka */ @ToString -public class ComponentLifecycleMsg implements TbActorMsg, TenantAwareMsg, ToAllNodesMsg { +public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { @Getter private final TenantId tenantId; @Getter diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java similarity index 71% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java index 56f928a1c8..0e8d66aa63 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java @@ -13,23 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.common.msg.queue; import lombok.Data; +import lombok.Getter; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import java.util.Set; + /** * @author Andrew Shvayka */ @Data -public final class ClusterEventMsg implements TbActorMsg { +public final class PartitionChangeMsg implements TbActorMsg { - private final ServerAddress serverAddress; - private final boolean added; + @Getter + private final ServiceQueueKey serviceQueueKey; + @Getter + private final Set partitions; @Override public MsgType getMsgType() { - return MsgType.CLUSTER_EVENT_MSG; + return MsgType.PARTITION_CHANGE_MSG; } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java similarity index 73% rename from common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java index a717af85a4..fca1c51995 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.system; +package org.thingsboard.server.common.msg.queue; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; @@ -22,18 +22,25 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import java.io.Serializable; +import java.util.Set; /** * Created by ashvayka on 15.03.18. */ @Data -public final class ServiceToRuleEngineMsg implements TbActorMsg, Serializable { +public final class QueueToRuleEngineMsg implements TbActorMsg { private final TenantId tenantId; private final TbMsg tbMsg; + private final Set relationTypes; + private final String failureMessage; @Override public MsgType getMsgType() { - return MsgType.SERVICE_TO_RULE_ENGINE_MSG; + return MsgType.QUEUE_TO_RULE_ENGINE_MSG; + } + + public boolean isTellNext() { + return relationTypes != null && !relationTypes.isEmpty(); } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java new file mode 100644 index 0000000000..dd720251bb --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RuleEngineException extends Exception { + protected static final ObjectMapper mapper = new ObjectMapper(); + + public RuleEngineException(String message) { + super(message != null ? message : "Unknown"); + } + + public String toJsonString() { + try { + return mapper.writeValueAsString(mapper.createObjectNode().put("message", getMessage())); + } catch (JsonProcessingException e) { + log.warn("Failed to serialize exception ", e); + throw new RuntimeException(e); + } + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java new file mode 100644 index 0000000000..3f437dd1d7 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.rule.RuleNode; + +@Slf4j +public class RuleNodeException extends RuleEngineException { + @Getter + private final String ruleChainName; + @Getter + private final String ruleNodeName; + @Getter + private final RuleChainId ruleChainId; + @Getter + private final RuleNodeId ruleNodeId; + + public RuleNodeException(String message, String ruleChainName, RuleNode ruleNode) { + super(message); + this.ruleChainName = ruleChainName; + this.ruleNodeName = ruleNode.getName(); + this.ruleChainId = ruleNode.getRuleChainId(); + this.ruleNodeId = ruleNode.getId(); + } + + public String toJsonString() { + try { + return mapper.writeValueAsString(mapper.createObjectNode() + .put("ruleNodeId", ruleNodeId.toString()) + .put("ruleChainId", ruleChainId.toString()) + .put("ruleNodeName", ruleNodeName) + .put("ruleChainName", ruleChainName) + .put("message", getMessage())); + } catch (JsonProcessingException e) { + log.warn("Failed to serialize exception ", e); + throw new RuntimeException(e); + } + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java new file mode 100644 index 0000000000..be08f054b6 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import lombok.ToString; + +import java.util.Objects; + +@ToString +public class ServiceQueue { + + public static final String MAIN = "Main"; + + private final ServiceType type; + private final String queue; + + public ServiceQueue(ServiceType type) { + this.type = type; + this.queue = MAIN; + } + + public ServiceQueue(ServiceType type, String queue) { + this.type = type; + this.queue = queue; + } + + public ServiceType getType() { + return type; + } + + public String getQueue() { + return queue; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceQueue that = (ServiceQueue) o; + return type == that.type && + queue.equals(that.queue); + } + + @Override + public int hashCode() { + return Objects.hash(type, queue); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java similarity index 50% rename from application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java index 570112dab3..f4bfefd878 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java @@ -13,35 +13,42 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.rpc; +package org.thingsboard.server.common.msg.queue; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.ToString; -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; -/** - * Created by ashvayka on 16.04.18. - */ +import java.util.Objects; + @ToString -@RequiredArgsConstructor -public class ToServerRpcResponseActorMsg implements ToDeviceActorNotificationMsg { +public class ServiceQueueKey { + @Getter + private final ServiceQueue serviceQueue; @Getter private final TenantId tenantId; - @Getter - private final DeviceId deviceId; + public ServiceQueueKey(ServiceQueue serviceQueue, TenantId tenantId) { + this.serviceQueue = serviceQueue; + this.tenantId = tenantId; + } - @Getter - private final ToServerRpcResponseMsg msg; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceQueueKey that = (ServiceQueueKey) o; + return serviceQueue.equals(that.serviceQueue) && + Objects.equals(tenantId, that.tenantId); + } @Override - public MsgType getMsgType() { - return MsgType.SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG; + public int hashCode() { + return Objects.hash(serviceQueue, tenantId); + } + + public ServiceType getServiceType() { + return serviceQueue.getType(); } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java new file mode 100644 index 0000000000..0ca776126d --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +public enum ServiceType { + + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; + + public static ServiceType of(String serviceType) { + return ServiceType.valueOf(serviceType.replace("-", "_").toUpperCase()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java similarity index 66% rename from application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java index d0ba27814c..4c25f27a3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java @@ -13,21 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.discovery; +package org.thingsboard.server.common.msg.queue; -import java.util.List; +public interface TbCallback { -/** - * @author Andrew Shvayka - */ -public interface DiscoveryService { + TbCallback EMPTY = new TbCallback() { + + @Override + public void onSuccess() { + + } - void publishCurrentServer(); + @Override + public void onFailure(Throwable t) { - void unpublishCurrentServer(); + } + }; - ServerInstance getCurrentServer(); + void onSuccess(); - List getOtherServers(); + void onFailure(Throwable t); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java similarity index 63% rename from common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java index 5d57514275..9e8d5ae6b8 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java @@ -13,18 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg; +package org.thingsboard.server.common.msg.queue; -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; +public interface TbMsgCallback { -import java.io.Serializable; -import java.util.UUID; + TbMsgCallback EMPTY = new TbMsgCallback() { -@Data -public final class TbMsgTransactionData implements Serializable { + @Override + public void onSuccess() { - private final UUID transactionId; - private final EntityId originatorId; + } + + @Override + public void onFailure(RuleEngineException e) { + + } + }; + + void onSuccess(); + + void onFailure(RuleEngineException e); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java new file mode 100644 index 0000000000..bf0fffe404 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; +import java.util.Optional; + +@ToString +public class TopicPartitionInfo { + + private final String topic; + private final TenantId tenantId; + private final Integer partition; + @Getter + private final String fullTopicName; + @Getter + private final boolean myPartition; + + @Builder + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { + this.topic = topic; + this.tenantId = tenantId; + this.partition = partition; + this.myPartition = myPartition; + String tmp = topic; + if (tenantId != null) { + tmp += "." + tenantId.getId().toString(); + } + if (partition != null) { + tmp += "." + partition; + } + this.fullTopicName = tmp; + } + + public String getTopic() { + return topic; + } + + public Optional getTenantId() { + return Optional.ofNullable(tenantId); + } + + public Optional getPartition() { + return Optional.ofNullable(partition); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TopicPartitionInfo that = (TopicPartitionInfo) o; + return topic.equals(that.topic) && + Objects.equals(tenantId, that.tenantId) && + Objects.equals(partition, that.partition) && + fullTopicName.equals(that.fullTopicName); + } + + @Override + public int hashCode() { + return Objects.hash(fullTopicName); + } +} diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 03dc17b5c9..737d5f6f57 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -23,13 +23,6 @@ message TbMsgMetaDataProto { map data = 1; } -message TbMsgTransactionDataProto { - string id = 1; - string entityType = 2; - int64 entityIdMSB = 3; - int64 entityIdLSB = 4; -} - message TbMsgProto { string id = 1; string type = 2; @@ -46,7 +39,7 @@ message TbMsgProto { TbMsgMetaDataProto metaData = 11; - TbMsgTransactionDataProto transactionData = 12; + //Transaction Data (12) was removed in 2.5 int32 dataType = 13; string data = 14; diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 8f0a0913a4..668da07e29 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -40,6 +40,10 @@ org.thingsboard.common data + + org.thingsboard.common + util + org.thingsboard.common message @@ -48,6 +52,22 @@ org.apache.kafka kafka-clients + + com.amazonaws + aws-java-sdk-sqs + + + com.google.cloud + google-cloud-pubsub + + + com.microsoft.azure + azure-servicebus + + + com.rabbitmq + amqp-client + org.springframework spring-context-support @@ -84,6 +104,14 @@ ch.qos.logback logback-classic + + com.google.protobuf + protobuf-java + + + org.apache.curator + curator-recipes + junit junit @@ -96,4 +124,13 @@ + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + + diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java deleted file mode 100644 index 0fe86e030b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import org.apache.kafka.clients.admin.*; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.KafkaFuture; - -import java.time.Duration; -import java.util.Collections; -import java.util.Properties; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * Created by ashvayka on 24.09.18. - */ -public class TBKafkaAdmin { - - AdminClient client; - - public TBKafkaAdmin(TbKafkaSettings settings) { - client = AdminClient.create(settings.toProps()); - } - - public void waitForTopic(String topic, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException { - synchronized (this) { - long timeoutExpiredMs = System.currentTimeMillis() + timeoutUnit.toMillis(timeout); - while (!topicExists(topic)) { - long waitMs = timeoutExpiredMs - System.currentTimeMillis(); - if (waitMs <= 0) { - throw new TimeoutException("Timeout occurred while waiting for topic [" + topic + "] to be available!"); - } else { - wait(1000); - } - } - } - } - - public CreateTopicsResult createTopic(NewTopic topic){ - return client.createTopics(Collections.singletonList(topic)); - } - - private boolean topicExists(String topic) throws InterruptedException { - KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); - try { - topicDescriptionFuture.get(); - return true; - } catch (ExecutionException e) { - return false; - } - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java deleted file mode 100644 index 73f8cc16c7..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Builder; -import lombok.Getter; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; - -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.Properties; -import java.util.UUID; - -/** - * Created by ashvayka on 24.09.18. - */ -public class TBKafkaConsumerTemplate { - - private final KafkaConsumer consumer; - private final TbKafkaDecoder decoder; - - @Builder.Default - private TbKafkaRequestIdExtractor requestIdExtractor = ((response) -> null); - - @Getter - private final String topic; - - @Builder - private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, - TbKafkaRequestIdExtractor requestIdExtractor, - String clientId, String groupId, String topic, - boolean autoCommit, int autoCommitIntervalMs, - int maxPollRecords) { - Properties props = settings.toProps(); - props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); - if (groupId != null) { - props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); - } - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit); - props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - if (maxPollRecords > 0) { - props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); - } - this.consumer = new KafkaConsumer<>(props); - this.decoder = decoder; - this.requestIdExtractor = requestIdExtractor; - this.topic = topic; - } - - public void subscribe() { - consumer.subscribe(Collections.singletonList(topic)); - } - - public void unsubscribe() { - consumer.unsubscribe(); - } - - public ConsumerRecords poll(Duration duration) { - return consumer.poll(duration); - } - - public T decode(ConsumerRecord record) throws IOException { - return decoder.decode(record.value()); - } - - public UUID extractRequestId(T value) { - return requestIdExtractor.extractRequestId(value); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java deleted file mode 100644 index 8722c1d64e..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Builder; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.KafkaFuture; -import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.errors.TopicExistsException; -import org.apache.kafka.common.header.Header; -import org.springframework.util.StringUtils; - -import java.util.List; -import java.util.Properties; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -/** - * Created by ashvayka on 24.09.18. - */ -@Slf4j -public class TBKafkaProducerTemplate { - - private final KafkaProducer producer; - private final TbKafkaEncoder encoder; - - private final TbKafkaPartitioner partitioner; - private ConcurrentMap> partitionInfoMap; - @Getter - private final String defaultTopic; - - @Getter - private final TbKafkaSettings settings; - - @Builder - private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaEncoder encoder, - TbKafkaPartitioner partitioner, String defaultTopic, String clientId) { - Properties props = settings.toProps(); - props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); - props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); - if (!StringUtils.isEmpty(clientId)) { - props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); - } - this.settings = settings; - this.producer = new KafkaProducer<>(props); - this.encoder = encoder; - this.partitioner = partitioner; - this.defaultTopic = defaultTopic; - } - - public void init() { - this.partitionInfoMap = new ConcurrentHashMap<>(); - if (!StringUtils.isEmpty(defaultTopic)) { - try { - TBKafkaAdmin admin = new TBKafkaAdmin(this.settings); - admin.waitForTopic(defaultTopic, 30, TimeUnit.SECONDS); - log.info("[{}] Topic exists.", defaultTopic); - } catch (Exception e) { - log.info("[{}] Failed to wait for topic: {}", defaultTopic, e.getMessage(), e); - throw new RuntimeException(e); - } - //Maybe this should not be cached, but we don't plan to change size of partitions - this.partitionInfoMap.putIfAbsent(defaultTopic, producer.partitionsFor(defaultTopic)); - } - } - - public Future send(String key, T value, Callback callback) { - return send(key, value, null, callback); - } - - public Future send(String key, T value, Iterable
headers, Callback callback) { - return send(key, value, null, headers, callback); - } - - public Future send(String key, T value, Long timestamp, Iterable
headers, Callback callback) { - if (!StringUtils.isEmpty(this.defaultTopic)) { - return send(this.defaultTopic, key, value, timestamp, headers, callback); - } else { - throw new RuntimeException("Failed to send message! Default topic is not specified!"); - } - } - - public Future send(String topic, String key, T value, Iterable
headers, Callback callback) { - return send(topic, key, value, null, headers, callback); - } - - public Future send(String topic, String key, T value, Callback callback) { - return send(topic, key, value, null, null, callback); - } - - public Future send(String topic, String key, T value, Long timestamp, Iterable
headers, Callback callback) { - byte[] data = encoder.encode(value); - ProducerRecord record; - Integer partition = getPartition(topic, key, value, data); - record = new ProducerRecord<>(topic, partition, timestamp, key, data, headers); - return producer.send(record, callback); - } - - private Integer getPartition(String topic, String key, T value, byte[] data) { - if (partitioner == null) { - return null; - } else { - return partitioner.partition(topic, key, value, data, partitionInfoMap.computeIfAbsent(topic, producer::partitionsFor)); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java deleted file mode 100644 index 864182a552..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Builder; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.internals.RecordHeader; - -import java.time.Duration; -import java.util.Collections; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Created by ashvayka on 25.09.18. - */ -@Slf4j -public class TbKafkaResponseTemplate extends AbstractTbKafkaTemplate { - - private final TBKafkaConsumerTemplate requestTemplate; - private final TBKafkaProducerTemplate responseTemplate; - private final TbKafkaHandler handler; - private final ConcurrentMap pendingRequests; - private final ExecutorService loopExecutor; - private final ScheduledExecutorService timeoutExecutor; - private final ExecutorService callbackExecutor; - private final int maxPendingRequests; - private final long requestTimeout; - - private final long pollInterval; - private volatile boolean stopped = false; - private final AtomicInteger pendingRequestCount = new AtomicInteger(); - - @Builder - public TbKafkaResponseTemplate(TBKafkaConsumerTemplate requestTemplate, - TBKafkaProducerTemplate responseTemplate, - TbKafkaHandler handler, - long pollInterval, - long requestTimeout, - int maxPendingRequests, - ExecutorService executor) { - this.requestTemplate = requestTemplate; - this.responseTemplate = responseTemplate; - this.handler = handler; - this.pendingRequests = new ConcurrentHashMap<>(); - this.maxPendingRequests = maxPendingRequests; - this.pollInterval = pollInterval; - this.requestTimeout = requestTimeout; - this.callbackExecutor = executor; - this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); - this.loopExecutor = Executors.newSingleThreadExecutor(); - } - - public void init() { - this.responseTemplate.init(); - requestTemplate.subscribe(); - loopExecutor.submit(() -> { - while (!stopped) { - try { - while (pendingRequestCount.get() >= maxPendingRequests) { - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e) { - log.trace("Failed to wait until the server has capacity to handle new requests", e); - } - } - ConsumerRecords requests = requestTemplate.poll(Duration.ofMillis(pollInterval)); - requests.forEach(request -> { - Header requestIdHeader = request.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); - if (requestIdHeader == null) { - log.error("[{}] Missing requestId in header", request); - return; - } - UUID requestId = bytesToUuid(requestIdHeader.value()); - if (requestId == null) { - log.error("[{}] Missing requestId in header and body", request); - return; - } - Header responseTopicHeader = request.headers().lastHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER); - if (responseTopicHeader == null) { - log.error("[{}] Missing response topic in header", request); - return; - } - String responseTopic = bytesToString(responseTopicHeader.value()); - try { - pendingRequestCount.getAndIncrement(); - Request decodedRequest = requestTemplate.decode(request); - AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(decodedRequest), - response -> { - pendingRequestCount.decrementAndGet(); - reply(requestId, responseTopic, response); - }, - e -> { - pendingRequestCount.decrementAndGet(); - if (e.getCause() != null && e.getCause() instanceof TimeoutException) { - log.warn("[{}] Timedout to process the request: {}", requestId, request, e); - } else { - log.trace("[{}] Failed to process the request: {}", requestId, request, e); - } - }, - requestTimeout, - timeoutExecutor, - callbackExecutor); - } catch (Throwable e) { - pendingRequestCount.decrementAndGet(); - log.warn("[{}] Failed to process the request: {}", requestId, request, e); - } - }); - } catch (InterruptException ie) { - if (!stopped) { - log.warn("Fetching data from kafka was interrupted.", ie); - } - } catch (Throwable e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - public void stop() { - stopped = true; - if (timeoutExecutor != null) { - timeoutExecutor.shutdownNow(); - } - if (loopExecutor != null) { - loopExecutor.shutdownNow(); - } - } - - private void reply(UUID requestId, String topic, Response response) { - responseTemplate.send(topic, requestId.toString(), response, Collections.singletonList(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))), null); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java new file mode 100644 index 0000000000..bb541fb626 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +public interface TbQueueAdmin { + + void createTopicIfNotExists(String topic); + + void destroy(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java new file mode 100644 index 0000000000..f21ad80d10 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +public interface TbQueueCallback { + + void onSuccess(TbQueueMsgMetadata metadata); + + void onFailure(Throwable t); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java similarity index 63% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 8ec5cd1d97..1b2f984093 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -13,18 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue; -import org.apache.kafka.clients.producer.Partitioner; -import org.apache.kafka.common.PartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; +import java.util.Set; -/** - * Created by ashvayka on 25.09.18. - */ -public interface TbKafkaPartitioner extends Partitioner { +public interface TbQueueConsumer { + + String getTopic(); + + void subscribe(); + + void subscribe(Set partitions); + + void unsubscribe(); + + List poll(long durationInMillis); - int partition(String topic, String key, T value, byte[] encodedValue, List partitions); + void commit(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java index 5e5aa0591a..609cbd2107 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue; import com.google.common.util.concurrent.ListenableFuture; /** * Created by ashvayka on 05.10.18. */ -public interface TbKafkaHandler { +public interface TbQueueHandler { ListenableFuture handle(Request request); diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java similarity index 81% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java index 6e433e71a9..b0e4990305 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java @@ -13,12 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue; import java.util.UUID; -public interface TbKafkaRequestIdExtractor { +public interface TbQueueMsg { - UUID extractRequestId(T value); + UUID getKey(); + TbQueueMsgHeaders getHeaders(); + + byte[] getData(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java new file mode 100644 index 0000000000..4cb38f6685 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.thingsboard.server.queue.TbQueueMsg; + +public interface TbQueueMsgDecoder { + + T decode(TbQueueMsg msg) throws InvalidProtocolBufferException; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java new file mode 100644 index 0000000000..f205ed676f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +import java.util.Map; + +public interface TbQueueMsgHeaders { + + byte[] put(String key, byte[] value); + + byte[] get(String key); + + Map getData(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java new file mode 100644 index 0000000000..0c24fec438 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +public interface TbQueueMsgMetadata { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java new file mode 100644 index 0000000000..ef308291a9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +public interface TbQueueProducer { + + void init(); + + String getDefaultTopic(); + + void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback); + + void stop(); +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java similarity index 68% rename from application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java index ef6b565817..20530360d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.discovery; +package org.thingsboard.server.queue; -/** - * @author Andrew Shvayka - */ -public interface DiscoveryServiceListener { +import com.google.common.util.concurrent.ListenableFuture; + +public interface TbQueueRequestTemplate { + + void init(); - void onServerAdded(ServerInstance server); + ListenableFuture send(Request request); - void onServerUpdated(ServerInstance server); + void stop(); - void onServerRemoved(ServerInstance server); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java new file mode 100644 index 0000000000..c823b3df88 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +public interface TbQueueResponseTemplate { + + void init(TbQueueHandler handler); + + void stop(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java new file mode 100644 index 0000000000..336514b7b2 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -0,0 +1,77 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.microsoft.azure.servicebus.management.ManagementClient; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +@Slf4j +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") +public class TbServiceBusAdmin implements TbQueueAdmin { + + private final Set queues = ConcurrentHashMap.newKeySet(); + + private final ManagementClient client; + + public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings) { + ConnectionStringBuilder builder = new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + "queues", + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + + client = new ManagementClient(builder); + + try { + client.getQueues().forEach(queueDescription -> queues.add(queueDescription.getPath())); + } catch (ServiceBusException | InterruptedException e) { + log.error("Failed to get queues.", e); + throw new RuntimeException("Failed to get queues.", e); + } + } + + @Override + public void createTopicIfNotExists(String topic) { + if (queues.contains(topic)) { + return; + } + + try { + client.createQueue(topic); + queues.add(topic); + } catch (ServiceBusException | InterruptedException e) { + log.error("Failed to create queue: [{}]", topic, e); + } + } + + public void destroy() { + try { + client.close(); + } catch (IOException e) { + log.error("Failed to close ManagementClient."); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java new file mode 100644 index 0000000000..cca599d59a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java @@ -0,0 +1,210 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.microsoft.azure.servicebus.TransactionContext; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.CoreMessageReceiver; +import com.microsoft.azure.servicebus.primitives.MessageWithDeliveryTag; +import com.microsoft.azure.servicebus.primitives.MessagingEntityType; +import com.microsoft.azure.servicebus.primitives.MessagingFactory; +import com.microsoft.azure.servicebus.primitives.SettleModePair; +import lombok.extern.slf4j.Slf4j; +import org.apache.qpid.proton.amqp.messaging.Data; +import org.apache.qpid.proton.amqp.transport.ReceiverSettleMode; +import org.apache.qpid.proton.amqp.transport.SenderSettleMode; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.time.Duration; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +public class TbServiceBusConsumerTemplate implements TbQueueConsumer { + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbServiceBusSettings serviceBusSettings; + + private final Gson gson = new Gson(); + + private Set receivers; + private volatile Set partitions; + private volatile boolean subscribed; + private volatile boolean stopped = false; + private Map> pendingMessages = new ConcurrentHashMap<>(); + private volatile int messagesPerQueue; + + public TbServiceBusConsumerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.serviceBusSettings = serviceBusSettings; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + receivers.forEach(CoreMessageReceiver::closeAsync); + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + createReceivers(); + messagesPerQueue = receivers.size() / partitions.size(); + subscribed = true; + } + + List>> messageFutures = + receivers.stream() + .map(receiver -> receiver + .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) + .whenComplete((messages, err) -> { + if (!CollectionUtils.isEmpty(messages)) { + pendingMessages.put(receiver, messages); + } else if (err != null) { + log.error("Failed to receive messages.", err); + } + })) + .collect(Collectors.toList()); + try { + return fromList(messageFutures) + .get() + .stream() + .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) + .map(message -> { + try { + return decode(message); + } catch (InvalidProtocolBufferException e) { + log.error("Failed to parse message.", e); + throw new RuntimeException("Failed to parse message.", e); + } + }).collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Service Bus consumer is stopped.", topic); + } else { + log.error("Failed to receive messages", e); + } + } + } + return Collections.emptyList(); + } + + private void createReceivers() { + List> receiverFutures = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .map(queue -> { + MessagingFactory factory; + try { + factory = MessagingFactory.createFromConnectionStringBuilder(createConnection(queue)); + } catch (InterruptedException | ExecutionException e) { + log.error("Failed to create factory for the queue [{}]", queue); + throw new RuntimeException("Failed to create the factory", e); + } + + return CoreMessageReceiver.create(factory, queue, queue, 0, + new SettleModePair(SenderSettleMode.UNSETTLED, ReceiverSettleMode.SECOND), + MessagingEntityType.QUEUE); + }).collect(Collectors.toList()); + + try { + receivers = new HashSet<>(fromList(receiverFutures).get()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Service Bus consumer is stopped.", topic); + } else { + log.error("Failed to create receivers", e); + } + } + } + + private ConnectionStringBuilder createConnection(String queue) { + admin.createTopicIfNotExists(queue); + return new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + queue, + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + } + + private CompletableFuture> fromList(List> futures) { + CompletableFuture>[] arrayFuture = new CompletableFuture[futures.size()]; + futures.toArray(arrayFuture); + + return CompletableFuture + .allOf(arrayFuture) + .thenApply(v -> futures + .stream() + .map(CompletableFuture::join) + .collect(Collectors.toList())); + } + + @Override + public void commit() { + pendingMessages.forEach((receiver, msgs) -> + msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); + pendingMessages.clear(); + } + + private T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(new String(((Data) data.getMessage().getBody()).getValue().getArray()), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java new file mode 100644 index 0000000000..5d9a931378 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java @@ -0,0 +1,110 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.google.gson.Gson; +import com.microsoft.azure.servicebus.IMessage; +import com.microsoft.azure.servicebus.Message; +import com.microsoft.azure.servicebus.QueueClient; +import com.microsoft.azure.servicebus.ReceiveMode; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Slf4j +public class TbServiceBusProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final TbServiceBusSettings serviceBusSettings; + private final Map clients = new HashMap<>(); + private ExecutorService executorService; + + public TbServiceBusProducerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + this.serviceBusSettings = serviceBusSettings; + executorService = Executors.newSingleThreadExecutor(); + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + IMessage message = new Message(gson.toJson(new DefaultTbQueueMsg(msg))); + CompletableFuture future = getClient(tpi.getFullTopicName()).sendAsync(message); + future.whenCompleteAsync((success, err) -> { + if (err != null) { + callback.onFailure(err); + } else { + callback.onSuccess(null); + } + }, executorService); + } + + @Override + public void stop() { + clients.forEach((t, client) -> { + try { + client.close(); + } catch (ServiceBusException e) { + log.error("Failed to close QueueClient.", e); + } + }); + + if (executorService != null) { + executorService.shutdownNow(); + } + } + + private QueueClient getClient(String topic) { + return clients.computeIfAbsent(topic, k -> { + admin.createTopicIfNotExists(topic); + ConnectionStringBuilder builder = + new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + topic, + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + try { + return new QueueClient(builder, ReceiveMode.PEEKLOCK); + } catch (InterruptedException | ServiceBusException e) { + log.error("Failed to create new client for the Queue: [{}]", topic, e); + throw new RuntimeException("Failed to create new client for the Queue", e); + } + }); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java new file mode 100644 index 0000000000..d872dcec0b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") +@Component +@Data +public class TbServiceBusSettings { + @Value("${queue.service_bus.namespace_name}") + private String namespaceName; + @Value("${queue.service_bus.sas_key_name}") + private String sasKeyName; + @Value("${queue.service_bus.sas_key}") + private String sasKey; + @Value("${queue.service_bus.max_messages}") + private int maxMessages; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java similarity index 70% rename from common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index 0c68c8dd5d..ec3b0b537a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -13,19 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; - -import lombok.extern.slf4j.Slf4j; +package org.thingsboard.server.queue.common; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.UUID; -/** - * Created by ashvayka on 25.09.18. - */ -@Slf4j -public abstract class AbstractTbKafkaTemplate { +public class AbstractTbQueueTemplate { + protected static final String REQUEST_ID_HEADER = "requestId"; + protected static final String RESPONSE_TOPIC_HEADER = "responseTopic"; + protected static final String REQUEST_TIME = "requestTime"; + protected byte[] uuidToBytes(UUID uuid) { ByteBuffer buf = ByteBuffer.allocate(16); buf.putLong(uuid.getMostSignificantBits()); @@ -47,4 +45,14 @@ public abstract class AbstractTbKafkaTemplate { protected String bytesToString(byte[] data) { return new String(data, StandardCharsets.UTF_8); } + + protected static byte[] longToBytes(long x) { + ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); + longBuffer.putLong(0, x); + return longBuffer.array(); + } + + protected static long bytesToLong(byte[] bytes) { + return ByteBuffer.wrap(bytes).getLong(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java similarity index 98% rename from common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java index a01411e48f..aa87df031a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.common; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java similarity index 55% rename from application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java index df41a7e95f..7584e8c2d0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java @@ -13,23 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.common; -import io.grpc.stub.StreamObserver; import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.queue.TbQueueMsg; import java.util.UUID; -/** - * @author Andrew Shvayka - */ @Data -public final class RpcSessionCreateRequestMsg { +public class DefaultTbQueueMsg implements TbQueueMsg { + private final UUID key; + private final byte[] data; + private final DefaultTbQueueMsgHeaders headers; - private final UUID msgUid; - private final ServerAddress remoteAddress; - private final StreamObserver responseObserver; + public DefaultTbQueueMsg(TbQueueMsg msg) { + this.key = msg.getKey(); + this.data = msg.getData(); + DefaultTbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + msg.getHeaders().getData().forEach(headers::put); + this.headers = headers; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java new file mode 100644 index 0000000000..36c3dd0999 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import org.thingsboard.server.queue.TbQueueMsgHeaders; + +import java.util.HashMap; +import java.util.Map; + +public class DefaultTbQueueMsgHeaders implements TbQueueMsgHeaders { + + protected final Map data = new HashMap<>(); + + @Override + public byte[] put(String key, byte[] value) { + return data.put(key, value); + } + + @Override + public byte[] get(String key) { + return data.get(key); + } + + @Override + public Map getData() { + return data; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java similarity index 54% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 2a7bca119f..f02ae63441 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -13,26 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.common; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.TopicExistsException; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.internals.RecordHeader; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -41,15 +37,14 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeoutException; -/** - * Created by ashvayka on 25.09.18. - */ @Slf4j -public class TbKafkaRequestTemplate extends AbstractTbKafkaTemplate { +public class DefaultTbQueueRequestTemplate extends AbstractTbQueueTemplate + implements TbQueueRequestTemplate { - private final TBKafkaProducerTemplate requestTemplate; - private final TBKafkaConsumerTemplate responseTemplate; - private final ConcurrentMap> pendingRequests; + private final TbQueueAdmin queueAdmin; + private final TbQueueProducer requestTemplate; + private final TbQueueConsumer responseTemplate; + private final ConcurrentMap> pendingRequests; private final boolean internalExecutor; private final ExecutorService executor; private final long maxRequestTimeout; @@ -60,12 +55,14 @@ public class TbKafkaRequestTemplate extends AbstractTbKafkaTe private volatile boolean stopped = false; @Builder - public TbKafkaRequestTemplate(TBKafkaProducerTemplate requestTemplate, - TBKafkaConsumerTemplate responseTemplate, - long maxRequestTimeout, - long maxPendingRequests, - long pollInterval, - ExecutorService executor) { + public DefaultTbQueueRequestTemplate(TbQueueAdmin queueAdmin, + TbQueueProducer requestTemplate, + TbQueueConsumer responseTemplate, + long maxRequestTimeout, + long maxPendingRequests, + long pollInterval, + ExecutorService executor) { + this.queueAdmin = queueAdmin; this.requestTemplate = requestTemplate; this.responseTemplate = responseTemplate; this.pendingRequests = new ConcurrentHashMap<>(); @@ -81,20 +78,9 @@ public class TbKafkaRequestTemplate extends AbstractTbKafkaTe } } + @Override public void init() { - try { - TBKafkaAdmin admin = new TBKafkaAdmin(this.requestTemplate.getSettings()); - CreateTopicsResult result = admin.createTopic(new NewTopic(responseTemplate.getTopic(), 1, (short) 1)); - result.all().get(); - } catch (Exception e) { - if ((e instanceof TopicExistsException) || (e.getCause() != null && e.getCause() instanceof TopicExistsException)) { - log.trace("[{}] Topic already exists. ", responseTemplate.getTopic()); - } else { - log.info("[{}] Failed to create topic: {}", responseTemplate.getTopic(), e.getMessage(), e); - throw new RuntimeException(e); - } - - } + queueAdmin.createTopicIfNotExists(responseTemplate.getTopic()); this.requestTemplate.init(); tickTs = System.currentTimeMillis(); responseTemplate.subscribe(); @@ -102,44 +88,29 @@ public class TbKafkaRequestTemplate extends AbstractTbKafkaTe long nextCleanupMs = 0L; while (!stopped) { try { - ConsumerRecords responses = responseTemplate.poll(Duration.ofMillis(pollInterval)); - if (responses.count() > 0) { - log.trace("Polling responses completed, consumer records count [{}]", responses.count()); + List responses = responseTemplate.poll(pollInterval); + if (responses.size() > 0) { + log.trace("Polling responses completed, consumer records count [{}]", responses.size()); + } else { + continue; } responses.forEach(response -> { - log.trace("Received response to Kafka Template request: {}", response); - Header requestIdHeader = response.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); - Response decodedResponse = null; - UUID requestId = null; + byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); + UUID requestId; if (requestIdHeader == null) { - try { - decodedResponse = responseTemplate.decode(response); - requestId = responseTemplate.extractRequestId(decodedResponse); - } catch (IOException e) { - log.error("Failed to decode response", e); - } - } else { - requestId = bytesToUuid(requestIdHeader.value()); - } - if (requestId == null) { log.error("[{}] Missing requestId in header and body", response); } else { - log.trace("[{}] Response received", requestId); + requestId = bytesToUuid(requestIdHeader); + log.trace("[{}] Response received: {}", requestId, response); ResponseMetaData expectedResponse = pendingRequests.remove(requestId); if (expectedResponse == null) { log.trace("[{}] Invalid or stale request", requestId); } else { - try { - if (decodedResponse == null) { - decodedResponse = responseTemplate.decode(response); - } - expectedResponse.future.set(decodedResponse); - } catch (IOException e) { - expectedResponse.future.setException(e); - } + expectedResponse.future.set(response); } } }); + responseTemplate.commit(); tickTs = System.currentTimeMillis(); tickSize = pendingRequests.size(); if (nextCleanupMs < tickTs) { @@ -155,10 +126,6 @@ public class TbKafkaRequestTemplate extends AbstractTbKafkaTe }); nextCleanupMs = tickTs + maxRequestTimeout; } - } catch (InterruptException ie) { - if (!stopped) { - log.warn("Fetching data from kafka was interrupted.", ie); - } } catch (Throwable e) { log.warn("Failed to obtain responses from queue.", e); try { @@ -171,30 +138,46 @@ public class TbKafkaRequestTemplate extends AbstractTbKafkaTe }); } + @Override public void stop() { stopped = true; + + if (responseTemplate != null) { + responseTemplate.unsubscribe(); + } + + if (requestTemplate != null) { + requestTemplate.stop(); + } + if (internalExecutor) { executor.shutdownNow(); } } - public ListenableFuture post(String key, Request request) { + @Override + public ListenableFuture send(Request request) { if (tickSize > maxPendingRequests) { return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); } UUID requestId = UUID.randomUUID(); - List
headers = new ArrayList<>(2); - headers.add(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))); - headers.add(new RecordHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic()))); + request.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + request.getHeaders().put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic())); + request.getHeaders().put(REQUEST_TIME, longToBytes(System.currentTimeMillis())); SettableFuture future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime); - requestTemplate.send(key, request, headers, (metadata, exception) -> { - if (exception != null) { - log.trace("[{}] Failed to post the request", requestId, exception); - } else { - log.trace("[{}] Posted the request: {}", requestId, metadata); + log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime); + requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.trace("[{}] Request sent: {}", requestId, metadata); + } + + @Override + public void onFailure(Throwable t) { + pendingRequests.remove(requestId); + future.setException(t); } }); return future; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java new file mode 100644 index 0000000000..96891ee7d8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java @@ -0,0 +1,163 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class DefaultTbQueueResponseTemplate extends AbstractTbQueueTemplate + implements TbQueueResponseTemplate { + + private final TbQueueConsumer requestTemplate; + private final TbQueueProducer responseTemplate; + private final ConcurrentMap pendingRequests; + private final ExecutorService loopExecutor; + private final ScheduledExecutorService timeoutExecutor; + private final ExecutorService callbackExecutor; + private final int maxPendingRequests; + private final long requestTimeout; + + private final long pollInterval; + private volatile boolean stopped = false; + private final AtomicInteger pendingRequestCount = new AtomicInteger(); + + @Builder + public DefaultTbQueueResponseTemplate(TbQueueConsumer requestTemplate, + TbQueueProducer responseTemplate, + TbQueueHandler handler, + long pollInterval, + long requestTimeout, + int maxPendingRequests, + ExecutorService executor) { + this.requestTemplate = requestTemplate; + this.responseTemplate = responseTemplate; + this.pendingRequests = new ConcurrentHashMap<>(); + this.maxPendingRequests = maxPendingRequests; + this.pollInterval = pollInterval; + this.requestTimeout = requestTimeout; + this.callbackExecutor = executor; + this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); + this.loopExecutor = Executors.newSingleThreadExecutor(); + } + + @Override + public void init(TbQueueHandler handler) { + this.responseTemplate.init(); + requestTemplate.subscribe(); + loopExecutor.submit(() -> { + while (!stopped) { + try { + while (pendingRequestCount.get() >= maxPendingRequests) { + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e) { + log.trace("Failed to wait until the server has capacity to handle new requests", e); + } + } + List requests = requestTemplate.poll(pollInterval); + + if (requests.isEmpty()) { + continue; + } + + requests.forEach(request -> { + long currentTime = System.currentTimeMillis(); + long requestTime = bytesToLong(request.getHeaders().get(REQUEST_TIME)); + if (requestTime + requestTimeout >= currentTime) { + byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header", request); + return; + } + byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); + if (responseTopicHeader == null) { + log.error("[{}] Missing response topic in header", request); + return; + } + UUID requestId = bytesToUuid(requestIdHeader); + String responseTopic = bytesToString(responseTopicHeader); + try { + pendingRequestCount.getAndIncrement(); + AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), + response -> { + pendingRequestCount.decrementAndGet(); + response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + responseTemplate.send(TopicPartitionInfo.builder().topic(responseTopic).build(), response, null); + }, + e -> { + pendingRequestCount.decrementAndGet(); + if (e.getCause() != null && e.getCause() instanceof TimeoutException) { + log.warn("[{}] Timeout to process the request: {}", requestId, request, e); + } else { + log.trace("[{}] Failed to process the request: {}", requestId, request, e); + } + }, + requestTimeout, + timeoutExecutor, + callbackExecutor); + } catch (Throwable e) { + pendingRequestCount.decrementAndGet(); + log.warn("[{}] Failed to process the request: {}", requestId, request, e); + } + } + }); + requestTemplate.commit(); + } catch (Throwable e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + public void stop() { + stopped = true; + if (requestTemplate != null) { + requestTemplate.unsubscribe(); + } + if (responseTemplate != null) { + responseTemplate.stop(); + } + if (timeoutExecutor != null) { + timeoutExecutor.shutdownNow(); + } + if (loopExecutor != null) { + loopExecutor.shutdownNow(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java new file mode 100644 index 0000000000..f860d8dcc9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; + +import java.util.concurrent.atomic.AtomicInteger; + +public class MultipleTbQueueTbMsgCallbackWrapper implements TbQueueCallback { + + private final AtomicInteger tbQueueCallbackCount; + private final TbMsgCallback tbMsgCallback; + + public MultipleTbQueueTbMsgCallbackWrapper(int tbQueueCallbackCount, TbMsgCallback tbMsgCallback) { + this.tbQueueCallbackCount = new AtomicInteger(tbQueueCallbackCount); + this.tbMsgCallback = tbMsgCallback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (tbQueueCallbackCount.decrementAndGet() <= 0) { + tbMsgCallback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java new file mode 100644 index 0000000000..07417c4a9e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.queue.TbQueueMsgHeaders; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +public class TbProtoJsQueueMsg extends TbProtoQueueMsg { + + public TbProtoJsQueueMsg(UUID key, T value) { + super(key, value); + } + + public TbProtoJsQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { + super(key, value, headers); + } + + @Override + public byte[] getData() { + try { + return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java new file mode 100644 index 0000000000..2eb76950dc --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import lombok.Data; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; + +import java.util.UUID; + +@Data +public class TbProtoQueueMsg implements TbQueueMsg { + + private final UUID key; + protected final T value; + private final TbQueueMsgHeaders headers; + + public TbProtoQueueMsg(UUID key, T value) { + this(key, value, new DefaultTbQueueMsgHeaders()); + } + + public TbProtoQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { + this.key = key; + this.value = value; + this.headers = headers; + } + + @Override + public UUID getKey() { + return key; + } + + @Override + public TbQueueMsgHeaders getHeaders() { + return headers; + } + + @Override + public byte[] getData() { + return value.toByteArray(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java new file mode 100644 index 0000000000..8bd9c3516b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; + +public class TbQueueTbMsgCallbackWrapper implements TbQueueCallback { + + private final TbMsgCallback tbMsgCallback; + + public TbQueueTbMsgCallbackWrapper(TbMsgCallback tbMsgCallback) { + this.tbMsgCallback = tbMsgCallback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + tbMsgCallback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java similarity index 55% rename from common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java index 5d47425e43..f0039d0f38 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java @@ -13,21 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.timeout; +package org.thingsboard.server.queue.discovery; -import org.thingsboard.server.common.msg.MsgType; +import lombok.Getter; +import org.springframework.context.ApplicationEvent; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; -/** - * @author Andrew Shvayka - */ -public final class DeviceActorClientSideRpcTimeoutMsg extends TimeoutMsg { +import java.util.Set; - public DeviceActorClientSideRpcTimeoutMsg(Integer id, long timeout) { - super(id, timeout); - } - @Override - public MsgType getMsgType() { - return MsgType.DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG; +public class ClusterTopologyChangeEvent extends ApplicationEvent { + + @Getter + private final Set serviceQueueKeys; + + public ClusterTopologyChangeEvent(Object source, Set serviceQueueKeys) { + super(source); + this.serviceQueueKeys = serviceQueueKeys; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java similarity index 69% rename from application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java index 711677dc78..1b40545cde 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.routing; +package org.thingsboard.server.queue.discovery; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -25,11 +24,10 @@ import java.util.concurrent.ConcurrentSkipListMap; * Created by ashvayka on 23.09.18. */ @Slf4j -public class ConsistentHashCircle { - private final ConcurrentNavigableMap circle = - new ConcurrentSkipListMap<>(); +public class ConsistentHashCircle { + private final ConcurrentNavigableMap circle = new ConcurrentSkipListMap<>(); - public void put(long hash, ServerInstance instance) { + public void put(long hash, T instance) { circle.put(hash, instance); } @@ -45,7 +43,7 @@ public class ConsistentHashCircle { return circle.containsKey(hash); } - public ConcurrentNavigableMap tailMap(Long hash) { + public ConcurrentNavigableMap tailMap(Long hash) { return circle.tailMap(hash); } @@ -53,11 +51,11 @@ public class ConsistentHashCircle { return circle.firstKey(); } - public ServerInstance get(Long hash) { + public T get(Long hash) { return circle.get(hash); } public void log() { - circle.entrySet().forEach((e) -> log.debug("{} -> {}", e.getKey(), e.getValue().getServerAddress())); + circle.forEach((key, value) -> log.debug("{} -> {}", key, value)); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java new file mode 100644 index 0000000000..df25ea3ba9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -0,0 +1,339 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; +import org.thingsboard.server.common.msg.queue.ServiceQueue; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; + +import javax.annotation.PostConstruct; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class ConsistentHashPartitionService implements PartitionService { + + @Value("${queue.core.topic}") + private String coreTopic; + @Value("${queue.core.partitions:100}") + private Integer corePartitions; + @Value("${queue.partitions.hash_function_name:murmur3_128}") + private String hashFunctionName; + @Value("${queue.partitions.virtual_nodes_size:16}") + private Integer virtualNodesSize; + + private final ApplicationEventPublisher applicationEventPublisher; + private final TbServiceInfoProvider serviceInfoProvider; + private final TenantRoutingInfoService tenantRoutingInfoService; + private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); + private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); + + private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + private ConcurrentMap tpiCache = new ConcurrentHashMap<>(); + + private Map tbCoreNotificationTopics = new HashMap<>(); + private Map tbRuleEngineNotificationTopics = new HashMap<>(); + private List currentOtherServices; + + private HashFunction hashFunction; + + public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, TenantRoutingInfoService tenantRoutingInfoService, ApplicationEventPublisher applicationEventPublisher) { + this.serviceInfoProvider = serviceInfoProvider; + this.tenantRoutingInfoService = tenantRoutingInfoService; + this.applicationEventPublisher = applicationEventPublisher; + } + + @PostConstruct + public void init() { + this.hashFunction = forName(hashFunctionName); + partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions); + partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic); + } + + @Override + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + return resolve(new ServiceQueue(serviceType), tenantId, entityId); + } + + @Override + public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { + return resolve(new ServiceQueue(serviceType, queueName), tenantId, entityId); + } + + private TopicPartitionInfo resolve(ServiceQueue serviceQueue, TenantId tenantId, EntityId entityId) { + int hash = hashFunction.newHasher() + .putLong(entityId.getId().getMostSignificantBits()) + .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); + Integer partitionSize = partitionSizes.get(serviceQueue); + int partition; + if (partitionSize != null) { + partition = Math.abs(hash % partitionSize); + } else { + //TODO: In 2.6/3.1 this should not happen because all Rule Engine Queues will be in the DB and we always know their partition sizes. + partition = 0; + } + boolean isolatedTenant = isIsolated(serviceQueue, tenantId); + TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceQueue, isolatedTenant ? tenantId : null, partition); + return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceQueue, tenantId, partition)); + } + + @Override + public void recalculatePartitions(ServiceInfo currentService, List otherServices) { + logServiceInfo(currentService); + otherServices.forEach(this::logServiceInfo); + Map> circles = new HashMap<>(); + addNode(circles, currentService); + for (ServiceInfo other : otherServices) { + addNode(circles, other); + } + ConcurrentMap> oldPartitions = myPartitions; + TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService); + myPartitions = new ConcurrentHashMap<>(); + partitionSizes.forEach((type, size) -> { + ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(type, myIsolatedOrSystemTenantId); + for (int i = 0; i < size; i++) { + ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceQueueKey), i); + if (currentService.equals(serviceInfo)) { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(type, getSystemOrIsolatedTenantId(serviceInfo)); + myPartitions.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(i); + } + } + }); + myPartitions.forEach((serviceQueueKey, partitions) -> { + if (!partitions.equals(oldPartitions.get(serviceQueueKey))) { + log.info("[{}] NEW PARTITIONS: {}", serviceQueueKey, partitions); + Set tpiList = partitions.stream() + .map(partition -> buildTopicPartitionInfo(serviceQueueKey, partition)) + .collect(Collectors.toSet()); + applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, tpiList)); + } + }); + tpiCache.clear(); + + if (currentOtherServices == null) { + currentOtherServices = new ArrayList<>(otherServices); + } else { + Set changes = new HashSet<>(); + Map> currentMap = getServiceKeyListMap(currentOtherServices); + Map> newMap = getServiceKeyListMap(otherServices); + currentOtherServices = otherServices; + currentMap.forEach((key, list) -> { + if (!list.equals(newMap.get(key))) { + changes.add(key); + } + }); + currentMap.keySet().forEach(newMap::remove); + changes.addAll(newMap.keySet()); + if (!changes.isEmpty()) { + applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); + } + } + } + + @Override + public Set getAllServiceIds(ServiceType serviceType) { + Set result = new HashSet<>(); + ServiceInfo current = serviceInfoProvider.getServiceInfo(); + if (current.getServiceTypesList().contains(serviceType.name())) { + result.add(current.getServiceId()); + } + for (ServiceInfo serviceInfo : currentOtherServices) { + if (serviceInfo.getServiceTypesList().contains(serviceType.name())) { + result.add(serviceInfo.getServiceId()); + } + } + return result; + } + + @Override + public TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId) { + switch (serviceType) { + case TB_CORE: + return tbCoreNotificationTopics.computeIfAbsent(serviceId, + id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); + case TB_RULE_ENGINE: + return tbRuleEngineNotificationTopics.computeIfAbsent(serviceId, + id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); + default: + return buildNotificationsTopicPartitionInfo(serviceType, serviceId); + } + } + + private Map> getServiceKeyListMap(List services) { + final Map> currentMap = new HashMap<>(); + services.forEach(serviceInfo -> { + for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { + ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + for (TransportProtos.QueueInfo queue : serviceInfo.getRuleEngineQueuesList()) { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); + } + } else { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); + } + } + }); + return currentMap; + } + + private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { + return new TopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueueKey serviceQueueKey, int partition) { + return buildTopicPartitionInfo(serviceQueueKey.getServiceQueue(), serviceQueueKey.getTenantId(), partition); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueue serviceQueue, TenantId tenantId, int partition) { + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); + tpi.topic(partitionTopics.get(serviceQueue)); + tpi.partition(partition); + ServiceQueueKey myPartitionsSearchKey; + if (isIsolated(serviceQueue, tenantId)) { + tpi.tenantId(tenantId); + myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, tenantId); + } else { + myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, new TenantId(TenantId.NULL_UUID)); + } + List partitions = myPartitions.get(myPartitionsSearchKey); + if (partitions != null) { + tpi.myPartition(partitions.contains(partition)); + } else { + tpi.myPartition(false); + } + return tpi.build(); + } + + private boolean isIsolated(ServiceQueue serviceQueue, TenantId tenantId) { + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + return false; + } + TenantRoutingInfo routingInfo = tenantRoutingInfoMap.get(tenantId); + if (routingInfo == null) { + synchronized (tenantRoutingInfoMap) { + routingInfo = tenantRoutingInfoMap.get(tenantId); + if (routingInfo == null) { + routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); + tenantRoutingInfoMap.put(tenantId, routingInfo); + } + } + } + if (routingInfo == null) { + throw new RuntimeException("Tenant not found!"); + } + switch (serviceQueue.getType()) { + case TB_CORE: + return routingInfo.isIsolatedTbCore(); + case TB_RULE_ENGINE: + return routingInfo.isIsolatedTbRuleEngine(); + default: + return false; + } + } + + private void logServiceInfo(TransportProtos.ServiceInfo server) { + TenantId tenantId = getSystemOrIsolatedTenantId(server); + if (tenantId.isNullUid()) { + log.info("[{}] Found common server: [{}]", server.getServiceId(), server.getServiceTypesList()); + } else { + log.info("[{}][{}] Found specific server: [{}]", server.getServiceId(), tenantId, server.getServiceTypesList()); + } + } + + private TenantId getSystemOrIsolatedTenantId(TransportProtos.ServiceInfo serviceInfo) { + return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB())); + } + + private void addNode(Map> circles, ServiceInfo instance) { + TenantId tenantId = getSystemOrIsolatedTenantId(instance); + for (String serviceTypeStr : instance.getServiceTypesList()) { + ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + for (TransportProtos.QueueInfo queue : instance.getRuleEngineQueuesList()) { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), tenantId); + partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getPartitions()); + partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getTopic()); + for (int i = 0; i < virtualNodesSize; i++) { + circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); + } + } + } else { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), tenantId); + for (int i = 0; i < virtualNodesSize; i++) { + circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); + } + } + } + } + + private ServiceInfo resolveByPartitionIdx(ConsistentHashCircle circle, Integer partitionIdx) { + if (circle == null || circle.isEmpty()) { + return null; + } + Long hash = hashFunction.newHasher().putInt(partitionIdx).hash().asLong(); + if (!circle.containsKey(hash)) { + ConcurrentNavigableMap tailMap = circle.tailMap(hash); + hash = tailMap.isEmpty() ? + circle.firstKey() : tailMap.firstKey(); + } + return circle.get(hash); + } + + private HashCode hash(ServiceInfo instance, int i) { + return hashFunction.newHasher().putString(instance.getServiceId(), StandardCharsets.UTF_8).putInt(i).hash(); + } + + public static HashFunction forName(String name) { + switch (name) { + case "murmur3_32": + return Hashing.murmur3_32(); + case "murmur3_128": + return Hashing.murmur3_128(); + case "crc32": + return Hashing.crc32(); + case "md5": + return Hashing.md5(); + default: + throw new IllegalArgumentException("Can't find hash function with name " + name); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java new file mode 100644 index 0000000000..1c60a03cdd --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import javax.annotation.PostConstruct; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; + +@Component +@Slf4j +public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { + + @Getter + @Value("${service.id:#{null}}") + private String serviceId; + + @Getter + @Value("${service.type:monolith}") + private String serviceType; + + @Getter + @Value("${service.tenant_id:}") + private String tenantIdStr; + + @Autowired(required = false) + private TbQueueRuleEngineSettings ruleEngineSettings; + + private List serviceTypes; + private ServiceInfo serviceInfo; + private TenantId isolatedTenant; + + @PostConstruct + public void init() { + if (StringUtils.isEmpty(serviceId)) { + try { + serviceId = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + serviceId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); + } + } + log.info("Current Service ID: {}", serviceId); + if (serviceType.equalsIgnoreCase("monolith")) { + serviceTypes = Collections.unmodifiableList(Arrays.asList(ServiceType.values())); + } else { + serviceTypes = Collections.singletonList(ServiceType.of(serviceType)); + } + ServiceInfo.Builder builder = ServiceInfo.newBuilder() + .setServiceId(serviceId) + .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())); + UUID tenantId; + if (!StringUtils.isEmpty(tenantIdStr)) { + tenantId = UUID.fromString(tenantIdStr); + isolatedTenant = new TenantId(tenantId); + } else { + tenantId = TenantId.NULL_UUID; + } + builder.setTenantIdMSB(tenantId.getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); + + if (serviceTypes.contains(ServiceType.TB_RULE_ENGINE) && ruleEngineSettings != null) { + for (TbRuleEngineQueueConfiguration queue : ruleEngineSettings.getQueues()) { + TransportProtos.QueueInfo queueInfo = TransportProtos.QueueInfo.newBuilder() + .setName(queue.getName()) + .setTopic(queue.getTopic()) + .setPartitions(queue.getPartitions()).build(); + builder.addRuleEngineQueues(queueInfo); + } + } + + serviceInfo = builder.build(); + } + + @Override + public ServiceInfo getServiceInfo() { + return serviceInfo; + } + + @Override + public boolean isService(ServiceType serviceType) { + return serviceTypes.contains(serviceType); + } + + @Override + public Optional getIsolatedTenant() { + return Optional.ofNullable(isolatedTenant); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java new file mode 100644 index 0000000000..36510261d6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +public interface DiscoveryService { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java index 009a7801e7..9017823efb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java @@ -13,55 +13,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.discovery; +package org.thingsboard.server.queue.discovery; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.annotation.DependsOn; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; import java.util.Collections; -import java.util.List; -/** - * @author Andrew Shvayka - */ @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) @Slf4j @DependsOn("environmentLogService") public class DummyDiscoveryService implements DiscoveryService { - @Autowired - private ServerInstanceService serverInstance; - - @PostConstruct - public void init() { - log.info("Initializing..."); - } - - @Override - public void publishCurrentServer() { - //Do nothing - } + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; - @Override - public void unpublishCurrentServer() { - //Do nothing - } - @Override - public ServerInstance getCurrentServer() { - return serverInstance.getSelf(); + public DummyDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.partitionService = partitionService; } - @Override - public List getOtherServers() { - return Collections.emptyList(); + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList()); } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java new file mode 100644 index 0000000000..19b8f665a8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +import java.util.Set; + + +public class PartitionChangeEvent extends ApplicationEvent { + + @Getter + private final ServiceQueueKey serviceQueueKey; + @Getter + private final Set partitions; + + public PartitionChangeEvent(Object source, ServiceQueueKey serviceQueueKey, Set partitions) { + super(source); + this.serviceQueueKey = serviceQueueKey; + this.partitions = partitions; + } + + public ServiceType getServiceType() { + return serviceQueueKey.getServiceQueue().getType(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java new file mode 100644 index 0000000000..e3b3e76b80 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.List; +import java.util.Set; + +/** + * Once application is ready or cluster topology changes, this Service will produce {@link PartitionChangeEvent} + */ +public interface PartitionService { + + TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + + TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId); + + /** + * Received from the Discovery service when network topology is changed. + * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} + * @param otherServices - all other discovered services {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} + */ + void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); + + /** + * Get all active service ids by service type + * @param serviceType to filter the list of services + * @return list of all active services + */ + Set getAllServiceIds(ServiceType serviceType); + + /** + * Each Service should start a consumer for messages that target individual service instance based on serviceId. + * This topic is likely to have single partition, and is always assigned to the service. + * @param serviceType + * @param serviceId + * @return + */ + TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId); +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 0d401e4d9c..fcc7c7a60d 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -13,19 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.queue.discovery; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; -import java.util.function.Consumer; +import java.util.Optional; -/** - * Created by ashvayka on 05.10.18. - */ -public interface RuleEngineTransportService { +public interface TbServiceInfoProvider { + + String getServiceId(); + + ServiceInfo getServiceInfo(); - void process(String nodeId, DeviceActorToTransportMsg msg); + boolean isService(ServiceType serviceType); - void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure); + Optional getIsolatedTenant(); } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java similarity index 71% rename from application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java index 811713819a..07c4c2d9c5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java @@ -13,18 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.discovery; -import akka.actor.ActorRef; import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ @Data -public final class SessionActorInfo { - protected final UUID sessionId; - protected final ActorRef actor; +public class TenantRoutingInfo { + private final TenantId tenantId; + private final boolean isolatedTbCore; + private final boolean isolatedTbRuleEngine; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java new file mode 100644 index 0000000000..685a36df30 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import org.thingsboard.server.common.data.id.TenantId; + +public interface TenantRoutingInfoService { + + TenantRoutingInfo getRoutingInfo(TenantId tenantId); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java new file mode 100644 index 0000000000..37661e85e4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.AllArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceQueue; + +import java.util.Objects; + +@AllArgsConstructor +public class TopicPartitionInfoKey { + private ServiceQueue serviceQueue; + private TenantId isolatedTenantId; + private int partition; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TopicPartitionInfoKey that = (TopicPartitionInfoKey) o; + return partition == that.partition && + serviceQueue.equals(that.serviceQueue) && + Objects.equals(isolatedTenantId, that.isolatedTenantId); + } + + @Override + public int hashCode() { + return Objects.hash(serviceQueue, isolatedTenantId, partition); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index e1cdd5f83e..e761a229c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -13,12 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.discovery; +package org.thingsboard.server.queue.discovery; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.SerializationException; -import org.apache.commons.lang3.SerializationUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.imps.CuratorFrameworkState; @@ -32,22 +30,14 @@ import org.apache.curator.retry.RetryForever; import org.apache.curator.utils.CloseableUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.utils.MiscUtils; +import org.thingsboard.server.gen.transport.TransportProtos; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -59,9 +49,6 @@ import java.util.stream.Collectors; import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED; -/** - * @author Andrew Shvayka - */ @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) @Slf4j @@ -78,42 +65,29 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi @Value("${zk.zk_dir}") private String zkDir; - private String zkNodesDir; - - @Autowired - private ServerInstanceService serverInstance; - - @Autowired - @Lazy - private TelemetrySubscriptionService tsSubService; - - @Autowired - @Lazy - private DeviceStateService deviceStateService; - - @Autowired - @Lazy - private ActorService actorService; - - @Autowired - @Lazy - private ClusterRoutingService routingService; + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; private ExecutorService reconnectExecutorService; - private CuratorFramework client; private PathChildrenCache cache; private String nodePath; + private String zkNodesDir; private volatile boolean stopped = true; + public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.partitionService = partitionService; + } + @PostConstruct public void init() { log.info("Initializing..."); - Assert.hasLength(zkUrl, MiscUtils.missingProperty("zk.url")); - Assert.notNull(zkRetryInterval, MiscUtils.missingProperty("zk.retry_interval_ms")); - Assert.notNull(zkConnectionTimeout, MiscUtils.missingProperty("zk.connection_timeout_ms")); - Assert.notNull(zkSessionTimeout, MiscUtils.missingProperty("zk.session_timeout_ms")); + Assert.hasLength(zkUrl, missingProperty("zk.url")); + Assert.notNull(zkRetryInterval, missingProperty("zk.retry_interval_ms")); + Assert.notNull(zkConnectionTimeout, missingProperty("zk.connection_timeout_ms")); + Assert.notNull(zkSessionTimeout, missingProperty("zk.session_timeout_ms")); reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); @@ -123,53 +97,47 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi initZkClient(); } - private void initZkClient() { - try { - client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); - client.start(); - client.blockUntilConnected(); - cache = new PathChildrenCache(client, zkNodesDir, true); - cache.getListenable().addListener(this); - cache.start(); - stopped = false; - log.info("ZK client connected"); - } catch (Exception e) { - log.error("Failed to connect to ZK: {}", e.getMessage(), e); - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - throw new RuntimeException(e); - } - } - - private void destroyZkClient() { - stopped = true; - try { - unpublishCurrentServer(); - } catch (Exception e) {} - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - log.info("ZK client disconnected"); + private List getOtherServers() { + return cache.getCurrentData().stream() + .filter(cd -> !cd.getPath().equals(nodePath)) + .map(cd -> { + try { + return TransportProtos.ServiceInfo.parseFrom(cd.getData()); + } catch (NoSuchElementException | InvalidProtocolBufferException e) { + log.error("Failed to decode ZK node", e); + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); } - @PreDestroy - public void destroy() { - destroyZkClient(); - reconnectExecutorService.shutdownNow(); - log.info("Stopped discovery service"); + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + if (stopped) { + log.debug("Ignoring application ready event. Service is stopped."); + return; + } else { + log.info("Received application ready event. Starting current ZK node."); + } + if (client.getState() != CuratorFrameworkState.STARTED) { + log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); + return; + } + publishCurrentServer(); + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); } - @Override public synchronized void publishCurrentServer() { - ServerInstance self = this.serverInstance.getSelf(); + TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); if (currentServerExists()) { - log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); + log.info("[{}] ZK node for current instance already exists, NOT created new one: {}", self.getServiceId(), nodePath); } else { try { - log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); + log.info("[{}] Creating ZK node for current instance", self.getServiceId()); nodePath = client.create() .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); - log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); + .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", self.toByteArray()); + log.info("[{}] Created ZK node for current instance: {}", self.getServiceId(), nodePath); client.getConnectionStateListenable().addListener(checkReconnect(self)); } catch (Exception e) { log.error("Failed to create ZK node", e); @@ -183,10 +151,10 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return false; } try { - ServerInstance self = this.serverInstance.getSelf(); - ServerAddress registeredServerAdress = null; - registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); - if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { + TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); + TransportProtos.ServiceInfo registeredServerInfo = null; + registeredServerInfo = TransportProtos.ServiceInfo.parseFrom(client.getData().forPath(nodePath)); + if (self.equals(registeredServerInfo)) { return true; } } catch (KeeperException.NoNodeException e) { @@ -197,9 +165,9 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return false; } - private ConnectionStateListener checkReconnect(ServerInstance self) { + private ConnectionStateListener checkReconnect(TransportProtos.ServiceInfo self) { return (client, newState) -> { - log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); + log.info("[{}] ZK state changed: {}", self.getServiceId(), newState); if (newState == ConnectionState.LOST) { reconnectExecutorService.submit(this::reconnect); } @@ -223,8 +191,25 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } - @Override - public void unpublishCurrentServer() { + private void initZkClient() { + try { + client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); + client.start(); + client.blockUntilConnected(); + cache = new PathChildrenCache(client, zkNodesDir, true); + cache.getListenable().addListener(this); + cache.start(); + stopped = false; + log.info("ZK client connected"); + } catch (Exception e) { + log.error("Failed to connect to ZK: {}", e.getMessage(), e); + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + throw new RuntimeException(e); + } + } + + private void unpublishCurrentServer() { try { if (nodePath != null) { client.delete().forPath(nodePath); @@ -235,41 +220,26 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } - @Override - public ServerInstance getCurrentServer() { - return serverInstance.getSelf(); + private void destroyZkClient() { + stopped = true; + try { + unpublishCurrentServer(); + } catch (Exception e) { + } + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + log.info("ZK client disconnected"); } - @Override - public List getOtherServers() { - return cache.getCurrentData().stream() - .filter(cd -> !cd.getPath().equals(nodePath)) - .map(cd -> { - try { - return new ServerInstance((ServerAddress) SerializationUtils.deserialize(cd.getData())); - } catch (NoSuchElementException e) { - log.error("Failed to decode ZK node", e); - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); + @PreDestroy + public void destroy() { + destroyZkClient(); + reconnectExecutorService.shutdownNow(); + log.info("Stopped discovery service"); } - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting current ZK node."); - if (stopped) { - log.debug("Ignoring application ready event. Service is stopped."); - return; - } - if (client.getState() != CuratorFrameworkState.STARTED) { - log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); - return; - } - publishCurrentServer(); - getOtherServers().forEach( - server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) - ); + public static String missingProperty(String propertyName) { + return "The " + propertyName + " property need to be set!"; } @Override @@ -297,34 +267,23 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi log.debug("Ignoring event about current server {}", pathChildrenCacheEvent); return; } - ServerInstance instance; + TransportProtos.ServiceInfo instance; try { - ServerAddress serverAddress = SerializationUtils.deserialize(data.getData()); - instance = new ServerInstance(serverAddress); - } catch (SerializationException e) { + instance = TransportProtos.ServiceInfo.parseFrom(data.getData()); + } catch (InvalidProtocolBufferException e) { log.error("Failed to decode server instance for node {}", data.getPath(), e); throw e; } - log.info("Processing [{}] event for [{}:{}]", pathChildrenCacheEvent.getType(), instance.getHost(), instance.getPort()); + log.info("Processing [{}] event for [{}]", pathChildrenCacheEvent.getType(), instance.getServiceId()); switch (pathChildrenCacheEvent.getType()) { case CHILD_ADDED: - routingService.onServerAdded(instance); - tsSubService.onClusterUpdate(); - deviceStateService.onClusterUpdate(); - actorService.onServerAdded(instance); - break; case CHILD_UPDATED: - routingService.onServerUpdated(instance); - actorService.onServerUpdated(instance); - break; case CHILD_REMOVED: - routingService.onServerRemoved(instance); - tsSubService.onClusterUpdate(); - deviceStateService.onClusterUpdate(); - actorService.onServerRemoved(instance); + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); break; default: break; } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java index a6a881ecde..267b34b204 100644 --- a/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.environment; +package org.thingsboard.server.queue.environment; import lombok.extern.slf4j.Slf4j; import org.apache.zookeeper.Environment; @@ -33,7 +33,7 @@ public class EnvironmentLogService { @PostConstruct public void init() { - Environment.logEnv("Thingsboard server environment: ", log); + Environment.logEnv("ThingsBoard server environment: ", log); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java new file mode 100644 index 0000000000..98470eafba --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; + +import java.util.UUID; + +public class KafkaTbQueueMsg implements TbQueueMsg { + private final UUID key; + private final TbQueueMsgHeaders headers; + private final byte[] data; + + public KafkaTbQueueMsg(ConsumerRecord record) { + this.key = UUID.fromString(record.key()); + TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + record.headers().forEach(header -> { + headers.put(header.key(), header.value()); + }); + this.headers = headers; + this.data = record.value(); + } + + @Override + public UUID getKey() { + return key; + } + + @Override + public TbQueueMsgHeaders getHeaders() { + return headers; + } + + @Override + public byte[] getData() { + return data; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java index bb87d2ea88..5a9eaa632c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java @@ -13,17 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.kafka; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.thingsboard.server.queue.TbQueueMsgMetadata; -/** - * @author Andrew Shvayka - */ @Data -public final class RpcSessionClosedMsg { - - private final boolean client; - private final ServerAddress remoteAddress; +@AllArgsConstructor +public class KafkaTbQueueMsgMetadata implements TbQueueMsgMetadata { + private RecordMetadata metadata; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java new file mode 100644 index 0000000000..b92b094af1 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.common.errors.TopicExistsException; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; + +/** + * Created by ashvayka on 24.09.18. + */ +@Slf4j +public class TbKafkaAdmin implements TbQueueAdmin { + + private final AdminClient client; + private final Map topicConfigs; + private final Set topics = ConcurrentHashMap.newKeySet(); + + private final short replicationFactor; + + public TbKafkaAdmin(TbKafkaSettings settings, Map topicConfigs) { + client = AdminClient.create(settings.toProps()); + this.topicConfigs = topicConfigs; + + try { + topics.addAll(client.listTopics().names().get()); + } catch (InterruptedException | ExecutionException e) { + log.error("Failed to get all topics.", e); + } + + replicationFactor = settings.getReplicationFactor(); + } + + @Override + public void createTopicIfNotExists(String topic) { + if (topics.contains(topic)) { + return; + } + try { + NewTopic newTopic = new NewTopic(topic, 1, replicationFactor).configs(topicConfigs); + createTopic(newTopic).values().get(topic).get(); + topics.add(topic); + } catch (ExecutionException ee) { + if (ee.getCause() instanceof TopicExistsException) { + //do nothing + } else { + log.warn("[{}] Failed to create topic", topic, ee); + throw new RuntimeException(ee); + } + } catch (Exception e) { + log.warn("[{}] Failed to create topic", topic, e); + throw new RuntimeException(e); + } + + } + + @Override + public void destroy() { + if (client != null) { + client.close(); + } + } + + public CreateTopicsResult createTopic(NewTopic topic) { + return client.createTopics(Collections.singletonList(topic)); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java new file mode 100644 index 0000000000..49d1d74102 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -0,0 +1,159 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +/** + * Created by ashvayka on 24.09.18. + */ +@Slf4j +public class TbKafkaConsumerTemplate implements TbQueueConsumer { + + private final TbQueueAdmin admin; + private final KafkaConsumer consumer; + private final TbKafkaDecoder decoder; + private volatile boolean subscribed; + private volatile Set partitions; + private final Lock consumerLock; + + @Getter + private final String topic; + + @Builder + private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, + String clientId, String groupId, String topic, + boolean autoCommit, int autoCommitIntervalMs, + int maxPollRecords, + TbQueueAdmin admin) { + Properties props = settings.toProps(); + props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + if (groupId != null) { + props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + } + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit); + props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + if (maxPollRecords > 0) { + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); + } + this.admin = admin; + this.consumer = new KafkaConsumer<>(props); + this.decoder = decoder; + this.topic = topic; + this.consumerLock = new ReentrantLock(); + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + try { + consumerLock.lock(); + + if (!subscribed) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + topicNames.forEach(admin::createTopicIfNotExists); + consumer.subscribe(topicNames); + subscribed = true; + } + + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (records.count() > 0) { + List result = new ArrayList<>(); + records.forEach(record -> { + try { + result.add(decode(record)); + } catch (IOException e) { + log.error("Failed decode record: [{}]", record); + } + }); + return result; + } + } finally { + consumerLock.unlock(); + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + try { + consumerLock.lock(); + consumer.commitAsync(); + } finally { + consumerLock.unlock(); + } + } + + @Override + public void unsubscribe() { + try { + consumerLock.lock(); + if (consumer != null) { + consumer.unsubscribe(); + consumer.close(); + } + } finally { + consumerLock.unlock(); + } + } + + public T decode(ConsumerRecord record) throws IOException { + return decoder.decode(new KafkaTbQueueMsg(record)); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java similarity index 83% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java index ab196d5863..6e3ea1e4a9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java @@ -13,7 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; + +import org.thingsboard.server.queue.TbQueueMsg; import java.io.IOException; @@ -22,6 +24,6 @@ import java.io.IOException; */ public interface TbKafkaDecoder { - T decode(byte[] data) throws IOException; + T decode(TbQueueMsg msg) throws IOException; } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java index 22f65c58a0..b3c4dec8ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; /** * Created by ashvayka on 25.09.18. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java new file mode 100644 index 0000000000..4f26f51da7 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -0,0 +1,112 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; + +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * Created by ashvayka on 24.09.18. + */ +@Slf4j +public class TbKafkaProducerTemplate implements TbQueueProducer { + + private final KafkaProducer producer; + + @Getter + private final String defaultTopic; + + @Getter + private final TbKafkaSettings settings; + + private final TbQueueAdmin admin; + + private final Set topics; + + @Builder + private TbKafkaProducerTemplate(TbKafkaSettings settings, String defaultTopic, String clientId, TbQueueAdmin admin) { + Properties props = settings.toProps(); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + if (!StringUtils.isEmpty(clientId)) { + props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); + } + this.settings = settings; + this.producer = new KafkaProducer<>(props); + this.defaultTopic = defaultTopic; + this.admin = admin; + topics = ConcurrentHashMap.newKeySet(); + } + + @Override + public void init() { + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + createTopicIfNotExist(tpi); + String key = msg.getKey().toString(); + byte[] data = msg.getData(); + ProducerRecord record; + Iterable
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); + record = new ProducerRecord<>(tpi.getFullTopicName(), null, key, data, headers); + producer.send(record, (metadata, exception) -> { + if (exception == null) { + if (callback != null) { + callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); + } + } else { + if (callback != null) { + callback.onFailure(exception); + } else { + log.warn("Producer template failure: {}", exception.getMessage(), exception); + } + } + }); + } + + private void createTopicIfNotExist(TopicPartitionInfo tpi) { + if (topics.contains(tpi)) { + return; + } + admin.createTopicIfNotExists(tpi.getFullTopicName()); + topics.add(tpi); + } + + @Override + public void stop() { + if (producer != null) { + producer.close(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java new file mode 100644 index 0000000000..b6de7db50d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Data; + +/** + * Created by ashvayka on 25.09.18. + */ +@Data +public class TbKafkaProperty { + + private String key; + private String value; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java similarity index 79% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 6446643a18..66121cb215 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -13,12 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import java.util.List; @@ -28,31 +29,32 @@ import java.util.Properties; * Created by ashvayka on 25.09.18. */ @Slf4j -@ConditionalOnProperty(prefix = "kafka", value = "enabled", havingValue = "true", matchIfMissing = false) +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") @Component public class TbKafkaSettings { - static final String REQUEST_ID_HEADER = "requestId"; - static final String RESPONSE_TOPIC_HEADER = "responseTopic"; - - @Value("${kafka.bootstrap.servers}") + @Value("${queue.kafka.bootstrap.servers}") private String servers; - @Value("${kafka.acks}") + @Value("${queue.kafka.acks}") private String acks; - @Value("${kafka.retries}") + @Value("${queue.kafka.retries}") private int retries; - @Value("${kafka.batch.size}") + @Value("${queue.kafka.batch.size}") private int batchSize; - @Value("${kafka.linger.ms}") + @Value("${queue.kafka.linger.ms}") private long lingerMs; - @Value("${kafka.buffer.memory}") + @Value("${queue.kafka.buffer.memory}") private long bufferMemory; + @Value("${queue.kafka.replication_factor}") + @Getter + private short replicationFactor; + @Value("${kafka.other:#{null}}") private List other; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java new file mode 100644 index 0000000000..4b68abf929 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public class TbKafkaTopicConfigs { + @Value("${queue.kafka.topic-properties.core}") + private String coreProperties; + @Value("${queue.kafka.topic-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.kafka.topic-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.kafka.topic-properties.notifications}") + private String notificationsProperties; + @Value("${queue.kafka.topic-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreConfigs; + @Getter + private Map ruleEngineConfigs; + @Getter + private Map transportApiConfigs; + @Getter + private Map notificationsConfigs; + @Getter + private Map jsExecutorConfigs; + + @PostConstruct + private void init() { + coreConfigs = getConfigs(coreProperties); + ruleEngineConfigs = getConfigs(ruleEngineProperties); + transportApiConfigs = getConfigs(transportApiProperties); + notificationsConfigs = getConfigs(notificationsProperties); + jsExecutorConfigs = getConfigs(jsExecutorProperties); + } + + private Map getConfigs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + return configs; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java new file mode 100644 index 0000000000..f51f5b8ae0 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.memory; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +@Slf4j +public final class InMemoryStorage { + private static InMemoryStorage instance; + private final ConcurrentHashMap> storage; + + private InMemoryStorage() { + storage = new ConcurrentHashMap<>(); + } + + public static InMemoryStorage getInstance() { + if (instance == null) { + synchronized (InMemoryStorage.class) { + if (instance == null) { + instance = new InMemoryStorage(); + } + } + } + return instance; + } + + public boolean put(String topic, TbQueueMsg msg) { + return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); + } + + public List get(String topic) throws InterruptedException { + if (storage.containsKey(topic)) { + List entities; + T first = (T) storage.get(topic).poll(); + if (first != null) { + entities = new ArrayList<>(); + entities.add(first); + List otherList = new ArrayList<>(); + storage.get(topic).drainTo(otherList, 999); + for (TbQueueMsg other : otherList) { + entities.add((T) other); + } + } else { + entities = Collections.emptyList(); + } + return entities; + } + return Collections.emptyList(); + } + + /** + * Used primarily for testing. + */ + public void cleanup() { + storage.clear(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java new file mode 100644 index 0000000000..a619eda132 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.memory; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class InMemoryTbQueueConsumer implements TbQueueConsumer { + private final InMemoryStorage storage = InMemoryStorage.getInstance(); + private volatile Set partitions; + private volatile boolean stopped; + private volatile boolean subscribed; + + public InMemoryTbQueueConsumer(String topic) { + this.topic = topic; + stopped = false; + } + + private final String topic; + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = true; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = true; + } + + @Override + public void unsubscribe() { + stopped = true; + } + + @Override + public List poll(long durationInMillis) { + if (subscribed) { + List messages = partitions + .stream() + .map(tpi -> { + try { + return storage.get(tpi.getFullTopicName()); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Queue was interrupted.", e); + } + return Collections.emptyList(); + } + }) + .flatMap(List::stream) + .map(msg -> (T) msg).collect(Collectors.toList()); + if (messages.size() > 0) { + return messages; + } + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Failed to sleep.", e); + } + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java new file mode 100644 index 0000000000..cfcd788a16 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.memory; + +import lombok.Data; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +@Data +public class InMemoryTbQueueProducer implements TbQueueProducer { + + private final InMemoryStorage storage = InMemoryStorage.getInstance(); + + private final String defaultTopic; + + public InMemoryTbQueueProducer(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + + @Override + public void init() { + + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + boolean result = storage.put(tpi.getFullTopicName(), msg); + if (result) { + if (callback != null) { + callback.onSuccess(null); + } + } else { + if (callback != null) { + callback.onFailure(new RuntimeException("Failure add msg to InMemoryQueue")); + } + } + } + + @Override + public void stop() { + + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java new file mode 100644 index 0000000000..76ff04c238 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -0,0 +1,169 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") +public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbAwsSqsSettings sqsSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public AwsSqsMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.sqsSettings = sqsSettings; + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java new file mode 100644 index 0000000000..a7349091bd --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -0,0 +1,158 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") +public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { + + private final TbAwsSqsSettings sqsSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsQueueAttributes sqsQueueAttributes) { + this.sqsSettings = sqsSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..3056eced2c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -0,0 +1,135 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") +public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbAwsSqsSettings sqsSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + + public AwsSqsTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsSettings sqsSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.sqsSettings = sqsSettings; + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java new file mode 100644 index 0000000000..351cd4058c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -0,0 +1,124 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbAwsSqsSettings sqsSettings; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public AwsSqsTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.sqsSettings = sqsSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbAwsSqsProducerTemplate> producerTemplate = + new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); + + TbAwsSqsConsumerTemplate> consumerTemplate = + new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, + transportApiSettings.getResponsesTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(transportApiAdmin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java new file mode 100644 index 0000000000..fbdbeaab0c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -0,0 +1,123 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Slf4j +@Component +@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") +public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + + public InMemoryMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java new file mode 100644 index 0000000000..2855e577ee --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + + public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + InMemoryTbQueueProducer> producerTemplate = + new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + + InMemoryTbQueueConsumer> consumerTemplate = + new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + + templateBuilder.queueAdmin(new TbQueueAdmin() { + @Override + public void createTopicIfNotExists(String topic) {} + + @Override + public void destroy() {} + }); + + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java new file mode 100644 index 0000000000..72c6f7b8b9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -0,0 +1,250 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import com.google.protobuf.util.JsonFormat; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import java.nio.charset.StandardCharsets; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") +public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.partitionService = partitionService; + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + String queueName = configuration.getName(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(ruleEngineSettings.getTopic()); + consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(ruleEngineAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getTopic()); + consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(transportApiSettings.getRequestsTopic()); + consumerBuilder.clientId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(transportApiAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); + requestBuilder.admin(transportApiAdmin); + return requestBuilder.build(); + } + + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); + + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + responseBuilder.admin(jsExecutorAdmin); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java new file mode 100644 index 0000000000..ca10830981 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -0,0 +1,221 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import com.google.protobuf.util.JsonFormat; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; + +import java.nio.charset.StandardCharsets; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") +public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { + + private final PartitionService partitionService; + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaTbCoreQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.partitionService = partitionService; + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getTopic()); + consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-notifications-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(transportApiSettings.getRequestsTopic()); + consumerBuilder.clientId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(transportApiAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); + + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + responseBuilder.admin(jsExecutorAdmin); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..bcac399a64 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -0,0 +1,193 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import com.google.protobuf.util.JsonFormat; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import java.nio.charset.StandardCharsets; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") +public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaTbRuleEngineQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.partitionService = partitionService; + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); + return requestBuilder.build(); + } + + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + String queueName = configuration.getName(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(ruleEngineSettings.getTopic()); + consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(ruleEngineAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-rule-engine-notifications-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + return consumerBuilder.build(); + } + + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); + + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + responseBuilder.admin(jsExecutorAdmin); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java new file mode 100644 index 0000000000..2f7aab4779 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -0,0 +1,138 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { + + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaTbTransportQueueFactory(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + requestBuilder.admin(transportApiAdmin); + + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("transport-api-response-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + responseBuilder.admin(transportApiAdmin); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(transportApiAdmin); + templateBuilder.requestTemplate(requestBuilder.build()); + templateBuilder.responseTemplate(responseBuilder.build()); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-node-rule-engine-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("transport-api-notifications-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + responseBuilder.admin(notificationAdmin); + return responseBuilder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java new file mode 100644 index 0000000000..af0b276c1a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -0,0 +1,177 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") +public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); + + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java new file mode 100644 index 0000000000..0a23d58e46 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -0,0 +1,149 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") +public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..79501130ed --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -0,0 +1,138 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") +public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + + public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java new file mode 100644 index 0000000000..7e859e02ab --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -0,0 +1,132 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class PubSubTransportQueueFactory implements TbTransportQueueFactory { + + private final TbPubSubSettings pubSubSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { + this.pubSubSettings = pubSubSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(transportApiAdmin); + templateBuilder.requestTemplate(producer); + templateBuilder.responseTemplate(consumer); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, + transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java new file mode 100644 index 0000000000..72f45e5276 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -0,0 +1,169 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") +public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbRabbitMqSettings rabbitMqSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbRabbitMqSettings rabbitMqSettings, + TbRabbitMqQueueArguments queueArguments) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.rabbitMqSettings = rabbitMqSettings; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java new file mode 100644 index 0000000000..d84bbedbd2 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -0,0 +1,158 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") +public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { + + private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbRabbitMqQueueArguments queueArguments) { + this.rabbitMqSettings = rabbitMqSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..a18f427fab --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -0,0 +1,136 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") +public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbRabbitMqSettings rabbitMqSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + + public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbRabbitMqSettings rabbitMqSettings, + TbRabbitMqQueueArguments queueArguments) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.rabbitMqSettings = rabbitMqSettings; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java new file mode 100644 index 0000000000..841e004b3f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -0,0 +1,129 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbRabbitMqSettings rabbitMqSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + + public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbRabbitMqSettings rabbitMqSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbRabbitMqQueueArguments queueArguments) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.rabbitMqSettings = rabbitMqSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbRabbitMqProducerTemplate> producerTemplate = + new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + + TbRabbitMqConsumerTemplate> consumerTemplate = + new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(transportApiAdmin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java new file mode 100644 index 0000000000..8b01c38607 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -0,0 +1,142 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") +public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + + public ServiceBusMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceBusSettings serviceBusSettings, + TbQueueAdmin admin) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java new file mode 100644 index 0000000000..523ac88bd1 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -0,0 +1,124 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") +public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { + + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin admin; + + public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { + this.serviceBusSettings = serviceBusSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..8a2f9b396b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -0,0 +1,107 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") +public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + + public ServiceBusTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbServiceBusSettings serviceBusSettings, + TbQueueAdmin admin) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java new file mode 100644 index 0000000000..33c5277ca5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + + public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceBusSettings serviceBusSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueAdmin admin) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbQueueProducer> producerTemplate = + new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + + TbQueueConsumer> consumerTemplate = + new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(admin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java new file mode 100644 index 0000000000..59a8f449b0 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -0,0 +1,102 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Core Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbCoreQueueFactory { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> createTransportNotificationsMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> createRuleEngineMsgProducer(); + + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> createRuleEngineNotificationsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreMsgProducer(); + + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreNotificationsMsgProducer(); + + /** + * Used to consume messages by TB Core Service + * + * @return + */ + TbQueueConsumer> createToCoreMsgConsumer(); + + /** + * Used to consume high priority messages by TB Core Service + * + * @return + */ + TbQueueConsumer> createToCoreNotificationsMsgConsumer(); + + /** + * Used to consume Transport API Calls + * + * @return + */ + TbQueueConsumer> createTransportApiRequestConsumer(); + + /** + * Used to push replies to Transport API Calls + * + * @return + */ + TbQueueProducer> createTransportApiResponseProducer(); + + TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java new file mode 100644 index 0000000000..9fe5de0bcc --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { + + private final TbCoreQueueFactory tbQueueProvider; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + + public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return toTransport; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return toRuleEngineNotifications; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java new file mode 100644 index 0000000000..34c6ee577f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for providing various Producers to other services. + */ +public interface TbQueueProducerProvider { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> getTransportNotificationsMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineMsgProducer(); + + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreNotificationsMsgProducer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java new file mode 100644 index 0000000000..69de7ca3e2 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") +public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { + + private final TbRuleEngineQueueFactory tbQueueProvider; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + + + public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return toTransport; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return toRuleEngineNotifications; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..561b3e84ea --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Core Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbRuleEngineQueueFactory { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> createTransportNotificationsMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> createRuleEngineMsgProducer(); + + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> createRuleEngineNotificationsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreMsgProducer(); + + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreNotificationsMsgProducer(); + + /** + * Used to consume messages by TB Core Service + * + * @return + * @param configuration + */ + //TODO 2.5 ybondarenko: make sure you use queueName to distinct consumers where necessary + TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration); + + /** + * Used to consume high priority messages by TB Core Service + * + * @return + */ + TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); + + TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java new file mode 100644 index 0000000000..dc1d2c449c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; + +public interface TbTransportQueueFactory { + + TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate(); + + TbQueueProducer> createRuleEngineMsgProducer(); + + TbQueueProducer> createTbCoreMsgProducer(); + + TbQueueConsumer> createTransportNotificationsConsumer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java new file mode 100644 index 0000000000..bced3f83ac --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") +public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { + + private final TbTransportQueueFactory tbQueueProvider; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + + public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java new file mode 100644 index 0000000000..1241370d05 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -0,0 +1,186 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.cloud.pubsub.v1.SubscriptionAdminClient; +import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; +import com.google.cloud.pubsub.v1.TopicAdminClient; +import com.google.cloud.pubsub.v1.TopicAdminSettings; +import com.google.protobuf.Duration; +import com.google.pubsub.v1.ListSubscriptionsRequest; +import com.google.pubsub.v1.ListTopicsRequest; +import com.google.pubsub.v1.ProjectName; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.Topic; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +@Slf4j +public class TbPubSubAdmin implements TbQueueAdmin { + private static final String ACK_DEADLINE = "ackDeadlineInSec"; + private static final String MESSAGE_RETENTION = "messageRetentionInSec"; + + private final TopicAdminClient topicAdminClient; + private final SubscriptionAdminClient subscriptionAdminClient; + + private final TbPubSubSettings pubSubSettings; + private final Set topicSet = ConcurrentHashMap.newKeySet(); + private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); + private final Map subscriptionProperties; + + public TbPubSubAdmin(TbPubSubSettings pubSubSettings, Map subscriptionSettings) { + this.pubSubSettings = pubSubSettings; + this.subscriptionProperties = subscriptionSettings; + + TopicAdminSettings topicAdminSettings; + try { + topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + } catch (IOException e) { + log.error("Failed to create TopicAdminSettings"); + throw new RuntimeException("Failed to create TopicAdminSettings."); + } + + SubscriptionAdminSettings subscriptionAdminSettings; + try { + subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + } catch (IOException e) { + log.error("Failed to create SubscriptionAdminSettings"); + throw new RuntimeException("Failed to create SubscriptionAdminSettings."); + } + + try { + topicAdminClient = TopicAdminClient.create(topicAdminSettings); + + ListTopicsRequest listTopicsRequest = + ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); + TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); + for (Topic topic : response.iterateAll()) { + topicSet.add(topic.getName()); + } + } catch (IOException e) { + log.error("Failed to get topics.", e); + throw new RuntimeException("Failed to get topics.", e); + } + + try { + subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings); + + ListSubscriptionsRequest listSubscriptionsRequest = + ListSubscriptionsRequest.newBuilder() + .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) + .build(); + SubscriptionAdminClient.ListSubscriptionsPagedResponse response = + subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); + + for (Subscription subscription : response.iterateAll()) { + subscriptionSet.add(subscription.getName()); + } + } catch (IOException e) { + log.error("Failed to get subscriptions.", e); + throw new RuntimeException("Failed to get subscriptions.", e); + } + } + + @Override + public void createTopicIfNotExists(String partition) { + ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), partition); + + if (topicSet.contains(topicName.toString())) { + createSubscriptionIfNotExists(partition, topicName); + return; + } + + ListTopicsRequest listTopicsRequest = + ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); + TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); + for (Topic topic : response.iterateAll()) { + if (topic.getName().contains(topicName.toString())) { + topicSet.add(topic.getName()); + createSubscriptionIfNotExists(partition, topicName); + return; + } + } + + topicAdminClient.createTopic(topicName); + topicSet.add(topicName.toString()); + log.info("Created new topic: [{}]", topicName.toString()); + createSubscriptionIfNotExists(partition, topicName); + } + + private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); + + if (subscriptionSet.contains(subscriptionName.toString())) { + return; + } + + ListSubscriptionsRequest listSubscriptionsRequest = + ListSubscriptionsRequest.newBuilder().setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()).build(); + SubscriptionAdminClient.ListSubscriptionsPagedResponse response = subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); + for (Subscription subscription : response.iterateAll()) { + if (subscription.getName().equals(subscriptionName.toString())) { + subscriptionSet.add(subscription.getName()); + return; + } + } + + Subscription.Builder subscriptionBuilder = Subscription + .newBuilder() + .setName(subscriptionName.toString()) + .setTopic(topicName.toString()); + + setAckDeadline(subscriptionBuilder); + setMessageRetention(subscriptionBuilder); + + subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); + subscriptionSet.add(subscriptionName.toString()); + log.info("Created new subscription: [{}]", subscriptionName.toString()); + } + + private void setAckDeadline(Subscription.Builder builder) { + if (subscriptionProperties.containsKey(ACK_DEADLINE)) { + builder.setAckDeadlineSeconds(Integer.parseInt(subscriptionProperties.get(ACK_DEADLINE))); + } + } + + private void setMessageRetention(Subscription.Builder builder) { + if (subscriptionProperties.containsKey(MESSAGE_RETENTION)) { + Duration duration = Duration + .newBuilder() + .setSeconds(Long.parseLong(subscriptionProperties.get(MESSAGE_RETENTION))) + .build(); + builder.setMessageRetentionDuration(duration); + } + } + + @Override + public void destroy() { + if (topicAdminClient != null) { + topicAdminClient.close(); + } + if (subscriptionAdminClient != null) { + subscriptionAdminClient.close(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java new file mode 100644 index 0000000000..5dc795739a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -0,0 +1,219 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; +import com.google.cloud.pubsub.v1.stub.SubscriberStub; +import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings; +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.pubsub.v1.AcknowledgeRequest; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.PullRequest; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.ReceivedMessage; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@Slf4j +public class TbPubSubConsumerTemplate implements TbQueueConsumer { + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbPubSubSettings pubSubSettings; + + private volatile boolean subscribed; + private volatile Set partitions; + private volatile Set subscriptionNames; + private final List acknowledgeRequests = new CopyOnWriteArrayList<>(); + + private ExecutorService consumerExecutor; + private final SubscriberStub subscriber; + private volatile boolean stopped; + + private volatile int messagesPerTopic; + + public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.pubSubSettings = pubSubSettings; + this.topic = topic; + this.decoder = decoder; + + try { + SubscriberStubSettings subscriberStubSettings = + SubscriberStubSettings.newBuilder() + .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) + .setTransportChannelProvider( + SubscriberStubSettings.defaultGrpcTransportProviderBuilder() + .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) + .build()) + .build(); + + this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); + } catch (IOException e) { + log.error("Failed to create subscriber.", e); + throw new RuntimeException("Failed to create subscriber.", e); + } + stopped = false; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + + if (subscriber != null) { + subscriber.close(); + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); + subscriptionNames.forEach(admin::createTopicIfNotExists); + consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); + messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); + subscribed = true; + } + List messages; + try { + messages = receiveMessages(); + if (!messages.isEmpty()) { + List result = new ArrayList<>(); + messages.forEach(msg -> { + try { + result.add(decode(msg.getMessage())); + } catch (InvalidProtocolBufferException e) { + log.error("Failed decode record: [{}]", msg); + } + }); + return result; + } + } catch (ExecutionException | InterruptedException e) { + if (stopped) { + log.info("[{}] Pub/Sub consumer is stopped.", topic); + } else { + log.error("Failed to receive messages", e); + } + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); + acknowledgeRequests.clear(); + } + + private List receiveMessages() throws ExecutionException, InterruptedException { + List>> result = subscriptionNames.stream().map(subscriptionId -> { + String subscriptionName = ProjectSubscriptionName.format(pubSubSettings.getProjectId(), subscriptionId); + PullRequest pullRequest = + PullRequest.newBuilder() + .setMaxMessages(messagesPerTopic) + .setReturnImmediately(false) // return immediately if messages are not available + .setSubscription(subscriptionName) + .build(); + + ApiFuture pullResponseApiFuture = subscriber.pullCallable().futureCall(pullRequest); + + return ApiFutures.transform(pullResponseApiFuture, pullResponse -> { + if (pullResponse != null && !pullResponse.getReceivedMessagesList().isEmpty()) { + List ackIds = new ArrayList<>(); + for (ReceivedMessage message : pullResponse.getReceivedMessagesList()) { + ackIds.add(message.getAckId()); + } + AcknowledgeRequest acknowledgeRequest = + AcknowledgeRequest.newBuilder() + .setSubscription(subscriptionName) + .addAllAckIds(ackIds) + .build(); + + acknowledgeRequests.add(acknowledgeRequest); + return pullResponse.getReceivedMessagesList(); + } + return null; + }, consumerExecutor); + + }).collect(Collectors.toList()); + + ApiFuture> transform = ApiFutures.transform(ApiFutures.allAsList(result), listMessages -> { + if (!CollectionUtils.isEmpty(listMessages)) { + return listMessages.stream().filter(Objects::nonNull).flatMap(List::stream).collect(Collectors.toList()); + } + return Collections.emptyList(); + }, consumerExecutor); + + return transform.get(); + } + + public T decode(PubsubMessage message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java new file mode 100644 index 0000000000..2cd2e1054e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -0,0 +1,134 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.cloud.pubsub.v1.Publisher; +import com.google.gson.Gson; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.PubsubMessage; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class TbPubSubProducerTemplate implements TbQueueProducer { + + private final Gson gson = new Gson(); + + private final String defaultTopic; + private final TbQueueAdmin admin; + private final TbPubSubSettings pubSubSettings; + + private final Map publisherMap = new ConcurrentHashMap<>(); + + private ExecutorService pubExecutor = Executors.newCachedThreadPool(); + + public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) { + this.defaultTopic = defaultTopic; + this.admin = admin; + this.pubSubSettings = pubSubSettings; + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + PubsubMessage.Builder pubsubMessageBuilder = PubsubMessage.newBuilder(); + pubsubMessageBuilder.setData(getMsg(msg)); + + Publisher publisher = getOrCreatePublisher(tpi.getFullTopicName()); + ApiFuture future = publisher.publish(pubsubMessageBuilder.build()); + + ApiFutures.addCallback(future, new ApiFutureCallback() { + public void onSuccess(String messageId) { + if (callback != null) { + callback.onSuccess(null); + } + } + + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }, pubExecutor); + } + + @Override + public void stop() { + publisherMap.forEach((k, v) -> { + if (v != null) { + try { + v.shutdown(); + v.awaitTermination(1, TimeUnit.SECONDS); + } catch (Exception e) { + log.error("Failed to shutdown PubSub client during destroy()", e); + } + } + }); + + if (pubExecutor != null) { + pubExecutor.shutdownNow(); + } + } + + private ByteString getMsg(T msg) { + String json = gson.toJson(new DefaultTbQueueMsg(msg)); + return ByteString.copyFrom(json.getBytes()); + } + + private Publisher getOrCreatePublisher(String topic) { + if (publisherMap.containsKey(topic)) { + return publisherMap.get(topic); + } else { + try { + admin.createTopicIfNotExists(topic); + ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), topic); + Publisher publisher = Publisher.newBuilder(topicName).setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + publisherMap.put(topic, publisher); + return publisher; + } catch (IOException e) { + log.error("Failed to create topic [{}].", topic, e); + throw new RuntimeException("Failed to create topic.", e); + } + } + + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java new file mode 100644 index 0000000000..d6c6d0e271 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.ServiceAccountCredentials; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.io.ByteArrayInputStream; +import java.io.IOException; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") +@Component +@Data +public class TbPubSubSettings { + + @Value("${queue.pubsub.project_id}") + private String projectId; + + @Value("${queue.pubsub.service_account}") + private String serviceAccount; + + @Value("${queue.pubsub.max_msg_size}") + private int maxMsgSize; + + @Value("${queue.pubsub.max_messages}") + private int maxMessages; + + private CredentialsProvider credentialsProvider; + + @PostConstruct + private void init() throws IOException { + ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream( + new ByteArrayInputStream(serviceAccount.getBytes())); + credentialsProvider = FixedCredentialsProvider.create(credentials); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java new file mode 100644 index 0000000000..e98f51ec7c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") +public class TbPubSubSubscriptionSettings { + @Value("${queue.pubsub.queue-properties.core}") + private String coreProperties; + @Value("${queue.pubsub.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.pubsub.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.pubsub.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.pubsub.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreSettings; + @Getter + private Map ruleEngineSettings; + @Getter + private Map transportApiSettings; + @Getter + private Map notificationsSettings; + @Getter + private Map jsExecutorSettings; + + @PostConstruct + private void init() { + coreSettings = getSettings(coreProperties); + ruleEngineSettings = getSettings(ruleEngineProperties); + transportApiSettings = getSettings(transportApiProperties); + notificationsSettings = getSettings(notificationsProperties); + jsExecutorSettings = getSettings(jsExecutorProperties); + } + + private Map getSettings(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + return configs; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java new file mode 100644 index 0000000000..3bef6deb84 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeoutException; + +@Slf4j +public class TbRabbitMqAdmin implements TbQueueAdmin { + + private final TbRabbitMqSettings rabbitMqSettings; + private final Channel channel; + private final Connection connection; + private final Map arguments; + + public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map arguments) { + this.rabbitMqSettings = rabbitMqSettings; + this.arguments = arguments; + + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + } + + @Override + public void createTopicIfNotExists(String topic) { + try { + channel.queueDeclare(topic, false, false, false, arguments); + } catch (IOException e) { + log.error("Failed to bind queue: [{}]", topic, e); + } + } + + @Override + public void destroy() { + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close Chanel.", e); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close Connection.", e); + } + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java new file mode 100644 index 0000000000..25d7719163 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java @@ -0,0 +1,173 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.GetResponse; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +@Slf4j +public class TbRabbitMqConsumerTemplate implements TbQueueConsumer { + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbRabbitMqSettings rabbitMqSettings; + private final Channel channel; + private final Connection connection; + + private volatile Set partitions; + private volatile boolean subscribed; + private volatile Set queues; + private volatile boolean stopped; + + public TbRabbitMqConsumerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.rabbitMqSettings = rabbitMqSettings; + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + stopped = false; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close the channel."); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close the connection."); + } + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + queues = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.toSet()); + + queues.forEach(admin::createTopicIfNotExists); + subscribed = true; + } + + List result = queues.stream() + .map(queue -> { + try { + return channel.basicGet(queue, false); + } catch (IOException e) { + log.error("Failed to get messages from queue: [{}]", queue); + throw new RuntimeException("Failed to get messages from queue.", e); + } + }).filter(Objects::nonNull).map(message -> { + try { + return decode(message); + } catch (InvalidProtocolBufferException e) { + log.error("Failed to decode message: [{}].", message); + throw new RuntimeException("Failed to decode message.", e); + } + }).collect(Collectors.toList()); + if (result.size() > 0) { + return result; + } + } + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Failed to wait.", e); + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + try { + channel.basicAck(0, true); + } catch (IOException e) { + log.error("Failed to ack messages.", e); + } + } + + public T decode(GetResponse message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(new String(message.getBody()), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java new file mode 100644 index 0000000000..91b46213a5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java @@ -0,0 +1,113 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import com.rabbitmq.client.AMQP; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeoutException; + +@Slf4j +public class TbRabbitMqProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final TbRabbitMqSettings rabbitMqSettings; + private ListeningExecutorService producerExecutor; + private final Channel channel; + private final Connection connection; + + public TbRabbitMqProducerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + this.rabbitMqSettings = rabbitMqSettings; + producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + AMQP.BasicProperties properties = new AMQP.BasicProperties(); + try { + channel.basicPublish(rabbitMqSettings.getExchangeName(), tpi.getFullTopicName(), properties, gson.toJson(new DefaultTbQueueMsg(msg)).getBytes()); + if (callback != null) { + callback.onSuccess(null); + } + } catch (IOException e) { + log.error("Failed publish message: [{}].", msg, e); + if (callback != null) { + callback.onFailure(e); + } + } + } + + @Override + public void stop() { + if (producerExecutor != null) { + producerExecutor.shutdownNow(); + } + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close the channel."); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close the connection."); + } + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java new file mode 100644 index 0000000000..eb73e7dce6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") +public class TbRabbitMqQueueArguments { + @Value("${queue.rabbitmq.queue-properties.core}") + private String coreProperties; + @Value("${queue.rabbitmq.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.rabbitmq.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.rabbitmq.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.rabbitmq.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreArgs; + @Getter + private Map ruleEngineArgs; + @Getter + private Map transportApiArgs; + @Getter + private Map notificationsArgs; + @Getter + private Map jsExecutorArgs; + + @PostConstruct + private void init() { + coreArgs = getArgs(coreProperties); + ruleEngineArgs = getArgs(ruleEngineProperties); + transportApiArgs = getArgs(transportApiProperties); + notificationsArgs = getArgs(notificationsProperties); + jsExecutorArgs = getArgs(jsExecutorProperties); + } + + private Map getArgs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String strValue = property.substring(delimiterPosition + 1); + configs.put(key, getObjectValue(strValue)); + } + return configs; + } + + private Object getObjectValue(String str) { + if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) { + return Boolean.valueOf(str); + } else if (isNumeric(str)) { + return getNumericValue(str); + } + return str; + } + + private Object getNumericValue(String str) { + if (str.contains(".")) { + return Double.valueOf(str); + } else { + return Long.valueOf(str); + } + } + + private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?"); + + public boolean isNumeric(String strNum) { + if (strNum == null) { + return false; + } + return PATTERN.matcher(strNum).matches(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java new file mode 100644 index 0000000000..e0156e6dc8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.rabbitmq.client.ConnectionFactory; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") +@Component +@Data +public class TbRabbitMqSettings { + @Value("${queue.rabbitmq.exchange_name:}") + private String exchangeName; + @Value("${queue.rabbitmq.host:}") + private String host; + @Value("${queue.rabbitmq.port:}") + private int port; + @Value("${queue.rabbitmq.virtual_host:}") + private String virtualHost; + @Value("${queue.rabbitmq.username:}") + private String username; + @Value("${queue.rabbitmq.password:}") + private String password; + @Value("${queue.rabbitmq.automatic_recovery_enabled:}") + private boolean automaticRecoveryEnabled; + @Value("${queue.rabbitmq.connection_timeout:}") + private int connectionTimeout; + @Value("${queue.rabbitmq.handshake_timeout:}") + private int handshakeTimeout; + + private ConnectionFactory connectionFactory; + + @PostConstruct + private void init() { + connectionFactory = new ConnectionFactory(); + connectionFactory.setHost(host); + connectionFactory.setPort(port); + connectionFactory.setVirtualHost(virtualHost); + connectionFactory.setUsername(username); + connectionFactory.setPassword(password); + connectionFactory.setAutomaticRecoveryEnabled(automaticRecoveryEnabled); + connectionFactory.setConnectionTimeout(connectionTimeout); + connectionFactory.setHandshakeTimeout(handshakeTimeout); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java similarity index 73% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index cd52948da8..1c260c754d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -13,20 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.settings; import lombok.Data; -import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; -/** - * Created by ashvayka on 25.09.18. - */ @Data -public class TbKafkaProperty { +@Component +public class TbQueueCoreSettings { + + @Value("${queue.core.topic}") + private String topic; - private String key; - private String value; + @Value("${queue.core.partitions}") + private int partitions; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java new file mode 100644 index 0000000000..cd492e87f1 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueRemoteJsInvokeSettings { + @Value("${queue.js.request_topic}") + private String requestTopic; + + @Value("${queue.js.response_topic_prefix}") + private String responseTopic; + + @Value("${queue.js.max_pending_requests}") + private long maxPendingRequests; + + @Value("${queue.js.response_poll_interval}") + private int responsePollInterval; + + @Value("${queue.js.response_auto_commit_interval}") + private int autoCommitInterval; + + @Value("${queue.js.max_requests_timeout}") + private long maxRequestsTimeout; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java new file mode 100644 index 0000000000..9dfe715ece --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Slf4j +@Data +@EnableAutoConfiguration +@Configuration +@ConfigurationProperties(prefix = "queue.rule-engine") +public class TbQueueRuleEngineSettings { + + private String topic; + private List queues; + + //TODO 2.5 ybondarenko: make sure the queue names are valid to all queue providers. + // See how they are used in TbRuleEngineQueueFactory.createToRuleEngineMsgConsumer and all producers + @PostConstruct + public void validate() { + queues.stream().filter(queue -> queue.getName().equals("Main")).findFirst().orElseThrow(() -> { + log.error("Main queue is not configured in thingsboard.yml"); + return new RuntimeException("No \"Main\" queue configured!"); + }); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java new file mode 100644 index 0000000000..5ee58ab17e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueTransportApiSettings { + + @Value("${queue.transport_api.requests_topic}") + private String requestsTopic; + + @Value("${queue.transport_api.responses_topic}") + private String responsesTopic; + + @Value("${queue.transport_api.max_pending_requests}") + private int maxPendingRequests; + + @Value("${queue.transport_api.max_requests_timeout}") + private int maxRequestsTimeout; + + @Value("${queue.transport_api.max_callback_threads}") + private int maxCallbackThreads; + + @Value("${queue.transport_api.request_poll_interval}") + private long requestPollInterval; + + @Value("${queue.transport_api.response_poll_interval}") + private long responsePollInterval; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java similarity index 63% rename from application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java index f4b14144fc..50da4f4a1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java @@ -13,21 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.executors; +package org.thingsboard.server.queue.settings; +import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import org.thingsboard.common.util.AbstractListeningExecutor; +@Data @Component -public class ClusterRpcCallbackExecutorService extends AbstractListeningExecutor { +public class TbQueueTransportNotificationSettings { - @Value("${actors.cluster.grpc_callback_thread_pool_size}") - private int grpcCallbackExecutorThreadPoolSize; + @Value("${queue.transport.notifications_topic}") + private String notificationsTopic; - @Override - protected int getThreadPollSize() { - return grpcCallbackExecutorThreadPoolSize; - } + @Value("${queue.transport.poll_interval}") + private long transportPollInterval; } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java index f0ff8ce9e1..0d21c59c9c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java @@ -13,15 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.settings; import lombok.Data; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -/** - * @author Andrew Shvayka - */ @Data -public final class RpcBroadcastMsg { - private final ClusterAPIProtos.ClusterMessage msg; +public class TbRuleEngineQueueAckStrategyConfiguration { + + private String type; + private int retries; + private double failurePercentage; + private long pauseBetweenRetries; + } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java similarity index 62% rename from application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java index 489a5bfa17..019a76953b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.settings; import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ @Data -public final class RpcSessionConnectedMsg { +public class TbRuleEngineQueueConfiguration { + + private String name; + private String topic; + private int pollInterval; + private int partitions; + private long packProcessingTimeout; + private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy; + private TbRuleEngineQueueAckStrategyConfiguration processingStrategy; - private final ServerAddress remoteAddress; - private final UUID id; } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java similarity index 77% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java index ed1bbf18b5..e39dc0c322 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java @@ -13,12 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport; +package org.thingsboard.server.queue.settings; -import org.thingsboard.server.common.data.Device; +import lombok.Data; -public interface SessionMsgProcessor { +@Data +public class TbRuleEngineQueueSubmitStrategyConfiguration { - void onDeviceAdded(Device device); + private String type; + private int batchSize; } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java index 14e7504637..eb4eaed7ed 100644 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java @@ -13,17 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.rpc; +package org.thingsboard.server.queue.sqs; +import com.amazonaws.http.SdkHttpMetadata; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.queue.TbQueueMsgMetadata; -/** - * @author Andrew Shvayka - */ @Data -public final class RpcSessionDisconnectedMsg { +@AllArgsConstructor +public class AwsSqsTbQueueMsgMetadata implements TbQueueMsgMetadata { - private final boolean client; - private final ServerAddress remoteAddress; + private final SdkHttpMetadata metadata; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java new file mode 100644 index 0000000000..8e293e6dff --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.CreateQueueRequest; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +public class TbAwsSqsAdmin implements TbQueueAdmin { + + private final Map attributes; + private final AmazonSQS sqsClient; + private final Set queues; + + public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings, Map attributes) { + this.attributes = attributes; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(awsCredentials)) + .withRegion(sqsSettings.getRegion()) + .build(); + + queues = sqsClient + .listQueues() + .getQueueUrls() + .stream() + .map(this::getQueueNameFromUrl) + .collect(Collectors.toCollection(ConcurrentHashMap::newKeySet)); + } + + @Override + public void createTopicIfNotExists(String topic) { + String queueName = topic.replaceAll("\\.", "_") + ".fifo"; + if (queues.contains(queueName)) { + return; + } + final CreateQueueRequest createQueueRequest = new CreateQueueRequest(queueName).withAttributes(attributes); + String queueUrl = sqsClient.createQueue(createQueueRequest).getQueueUrl(); + queues.add(getQueueNameFromUrl(queueUrl)); + } + + private String getQueueNameFromUrl(String queueUrl) { + int delimiterIndex = queueUrl.lastIndexOf("/"); + return queueUrl.substring(delimiterIndex + 1); + } + + @Override + public void destroy() { + if (sqsClient != null) { + sqsClient.shutdown(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java new file mode 100644 index 0000000000..3e71388844 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -0,0 +1,231 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; +import com.amazonaws.services.sqs.model.Message; +import com.amazonaws.services.sqs.model.ReceiveMessageRequest; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +public class TbAwsSqsConsumerTemplate implements TbQueueConsumer { + + private static final int MAX_NUM_MSGS = 10; + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final AmazonSQS sqsClient; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbAwsSqsSettings sqsSettings; + + private final List pendingMessages = new CopyOnWriteArrayList<>(); + private volatile Set queueUrls; + private volatile Set partitions; + private ListeningExecutorService consumerExecutor; + private volatile boolean subscribed; + private volatile boolean stopped = false; + + public TbAwsSqsConsumerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.sqsSettings = sqsSettings; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); + + this.sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credProvider) + .withRegion(sqsSettings.getRegion()) + .build(); + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + + if (sqsClient != null) { + sqsClient.shutdown(); + } + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); + consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1)); + subscribed = true; + } + + if (!pendingMessages.isEmpty()) { + log.warn("Present {} non committed messages.", pendingMessages.size()); + return Collections.emptyList(); + } + + List>> futureList = queueUrls + .stream() + .map(url -> poll(url, (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis))) + .collect(Collectors.toList()); + ListenableFuture>> futureResult = Futures.allAsList(futureList); + try { + return futureResult.get().stream() + .flatMap(List::stream) + .map(msg -> { + try { + return decode(msg); + } catch (IOException e) { + log.error("Failed to decode message: [{}]", msg); + return null; + } + }).filter(Objects::nonNull) + .collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Aws SQS consumer is stopped.", topic); + } else { + log.error("Failed to pool messages.", e); + } + } + } + return Collections.emptyList(); + } + + private ListenableFuture> poll(String url, int waitTimeSeconds) { + List>> result = new ArrayList<>(); + + for (int i = 0; i < sqsSettings.getThreadsPerTopic(); i++) { + result.add(consumerExecutor.submit(() -> { + ReceiveMessageRequest request = new ReceiveMessageRequest(); + request + .withWaitTimeSeconds(waitTimeSeconds) + .withMessageAttributeNames("headers") + .withQueueUrl(url) + .withMaxNumberOfMessages(MAX_NUM_MSGS); + return sqsClient.receiveMessage(request).getMessages(); + })); + } + return Futures.transform(Futures.allAsList(result), list -> { + if (!CollectionUtils.isEmpty(list)) { + return list.stream() + .flatMap(messageList -> { + if (!messageList.isEmpty()) { + this.pendingMessages.add(new AwsSqsMsgWrapper(url, messageList)); + return messageList.stream(); + } + return Stream.empty(); + }) + .collect(Collectors.toList()); + } + return Collections.emptyList(); + }, consumerExecutor); + } + + @Override + public void commit() { + pendingMessages.forEach(msg -> + consumerExecutor.submit(() -> { + List entries = msg.getMessages() + .stream() + .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) + .collect(Collectors.toList()); + sqsClient.deleteMessageBatch(msg.getUrl(), entries); + })); + + pendingMessages.clear(); + } + + public T decode(Message message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } + + @Data + private static class AwsSqsMsgWrapper { + private final String url; + private final List messages; + + public AwsSqsMsgWrapper(String url, List messages) { + this.url = url; + this.messages = messages; + } + } + + private String getQueueUrl(String topic) { + admin.createTopicIfNotExists(topic); + return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java new file mode 100644 index 0000000000..2d85539184 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.SendMessageRequest; +import com.amazonaws.services.sqs.model.SendMessageResult; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; + +@Slf4j +public class TbAwsSqsProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final AmazonSQS sqsClient; + private final Gson gson = new Gson(); + private final Map queueUrlMap = new ConcurrentHashMap<>(); + private final TbQueueAdmin admin; + private ListeningExecutorService producerExecutor; + + public TbAwsSqsProducerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); + + this.sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credProvider) + .withRegion(sqsSettings.getRegion()) + .build(); + + producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + SendMessageRequest sendMsgRequest = new SendMessageRequest(); + sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); + sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg))); + + sendMsgRequest.withMessageGroupId(msg.getKey().toString()); + ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); + + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(SendMessageResult result) { + if (callback != null) { + callback.onSuccess(new AwsSqsTbQueueMsgMetadata(result.getSdkHttpMetadata())); + } + } + + @Override + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }, producerExecutor); + } + + @Override + public void stop() { + if (producerExecutor != null) { + producerExecutor.shutdownNow(); + } + if (sqsClient != null) { + sqsClient.shutdown(); + } + } + + private String getQueueUrl(String topic) { + return queueUrlMap.computeIfAbsent(topic, k -> { + admin.createTopicIfNotExists(topic); + return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); + }); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java new file mode 100644 index 0000000000..70a9587bef --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.services.sqs.model.QueueAttributeName; +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") +public class TbAwsSqsQueueAttributes { + @Value("${queue.aws-sqs.queue-properties.core}") + private String coreProperties; + @Value("${queue.aws-sqs.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.aws-sqs.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.aws-sqs.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.aws-sqs.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreAttributes; + @Getter + private Map ruleEngineAttributes; + @Getter + private Map transportApiAttributes; + @Getter + private Map notificationsAttributes; + @Getter + private Map jsExecutorAttributes; + + private final Map defaultAttributes = new HashMap<>(); + + @PostConstruct + private void init() { + defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true"); + defaultAttributes.put(QueueAttributeName.ContentBasedDeduplication.toString(), "true"); + + coreAttributes = getConfigs(coreProperties); + ruleEngineAttributes = getConfigs(ruleEngineProperties); + transportApiAttributes = getConfigs(transportApiProperties); + notificationsAttributes = getConfigs(notificationsProperties); + jsExecutorAttributes = getConfigs(jsExecutorProperties); + } + + private Map getConfigs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + validateAttributeName(key); + configs.put(key, value); + } + configs.putAll(defaultAttributes); + return configs; + } + + private void validateAttributeName(String key) { + QueueAttributeName.fromValue(key); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java similarity index 50% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java index 343c5e6dd8..922a2b1062 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java @@ -13,39 +13,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.sqs; -import lombok.Getter; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; -import javax.annotation.PostConstruct; -import java.net.InetAddress; -import java.net.UnknownHostException; - -/** - * Created by ashvayka on 12.10.18. - */ @Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") @Component -public class TbNodeIdProvider { +@Data +public class TbAwsSqsSettings { + + @Value("${queue.aws_sqs.access_key_id}") + private String accessKeyId; + + @Value("${queue.aws_sqs.secret_access_key}") + private String secretAccessKey; - @Getter - @Value("${cluster.node_id:#{null}}") - private String nodeId; + @Value("${queue.aws_sqs.region}") + private String region; - @PostConstruct - public void init() { - if (StringUtils.isEmpty(nodeId)) { - try { - nodeId = InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - nodeId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); - } - } - log.info("Current NodeId: {}", nodeId); - } + @Value("${queue.aws_sqs.threads_per_topic}") + private int threadsPerTopic; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java new file mode 100644 index 0000000000..ca2316e174 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +public @interface TbCoreComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java new file mode 100644 index 0000000000..3b99c94dea --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") +public @interface TbRuleEngineComponent { +} diff --git a/application/src/main/proto/jsinvoke.proto b/common/queue/src/main/proto/jsinvoke.proto similarity index 100% rename from application/src/main/proto/jsinvoke.proto rename to common/queue/src/main/proto/jsinvoke.proto diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto new file mode 100644 index 0000000000..51bc7649be --- /dev/null +++ b/common/queue/src/main/proto/queue.proto @@ -0,0 +1,410 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; +package transport; + +option java_package = "org.thingsboard.server.gen.transport"; +option java_outer_classname = "TransportProtos"; + +message QueueInfo { + string name = 1; + string topic = 2; + int32 partitions = 3; +} + +/** + * Service Discovery Data Structures; + */ +message ServiceInfo { + string serviceId = 1; + repeated string serviceTypes = 2; + int64 tenantIdMSB = 3; + int64 tenantIdLSB = 4; + repeated QueueInfo ruleEngineQueues = 5; +} + +/** + * Transport Service Data Structures; + */ +message SessionInfoProto { + string nodeId = 1; + int64 sessionIdMSB = 2; + int64 sessionIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + int64 deviceIdMSB = 6; + int64 deviceIdLSB = 7; + string deviceName = 8; + string deviceType = 9; +} + +enum SessionEvent { + OPEN = 0; + CLOSED = 1; +} + +enum SessionType { + SYNC = 0; + ASYNC = 1; +} + +enum KeyValueType { + BOOLEAN_V = 0; + LONG_V = 1; + DOUBLE_V = 2; + STRING_V = 3; + JSON_V = 4; +} + +message KeyValueProto { + string key = 1; + KeyValueType type = 2; + bool bool_v = 3; + int64 long_v = 4; + double double_v = 5; + string string_v = 6; + string json_v = 7; +} + +message TsKvProto { + int64 ts = 1; + KeyValueProto kv = 2; +} + +message TsKvListProto { + int64 ts = 1; + repeated KeyValueProto kv = 2; +} + +message DeviceInfoProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + string deviceName = 5; + string deviceType = 6; + string additionalInfo = 7; +} + +/** + * Transport Service Messages; + */ +message SessionEventMsg { + SessionType sessionType = 1; + SessionEvent event = 2; +} + +message PostTelemetryMsg { + repeated TsKvListProto tsKvList = 1; +} + +message PostAttributeMsg { + repeated KeyValueProto kv = 1; +} + +message GetAttributeRequestMsg { + int32 requestId = 1; + repeated string clientAttributeNames = 2; + repeated string sharedAttributeNames = 3; +} + +message GetAttributeResponseMsg { + int32 requestId = 1; + repeated TsKvProto clientAttributeList = 2; + repeated TsKvProto sharedAttributeList = 3; + repeated string deletedAttributeKeys = 4; + string error = 5; +} + +message AttributeUpdateNotificationMsg { + repeated TsKvProto sharedUpdated = 1; + repeated string sharedDeleted = 2; +} + +message ValidateDeviceTokenRequestMsg { + string token = 1; +} + +message ValidateDeviceX509CertRequestMsg { + string hash = 1; +} + +message ValidateDeviceCredentialsResponseMsg { + DeviceInfoProto deviceInfo = 1; + string credentialsBody = 2; +} + +message GetOrCreateDeviceFromGatewayRequestMsg { + int64 gatewayIdMSB = 1; + int64 gatewayIdLSB = 2; + string deviceName = 3; + string deviceType = 4; +} + +message GetOrCreateDeviceFromGatewayResponseMsg { + DeviceInfoProto deviceInfo = 1; +} + +message GetTenantRoutingInfoRequestMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; +} + +message GetTenantRoutingInfoResponseMsg { + bool isolatedTbCore = 1; + bool isolatedTbRuleEngine = 2; +} + +message SessionCloseNotificationProto { + string message = 1; +} + +message SubscribeToAttributeUpdatesMsg { + bool unsubscribe = 1; +} + +message SubscribeToRPCMsg { + bool unsubscribe = 1; +} + +message ToDeviceRpcRequestMsg { + int32 requestId = 1; + string methodName = 2; + string params = 3; +} + +message ToDeviceRpcResponseMsg { + int32 requestId = 1; + string payload = 2; +} + +message ToServerRpcRequestMsg { + int32 requestId = 1; + string methodName = 2; + string params = 3; +} + +message ToServerRpcResponseMsg { + int32 requestId = 1; + string payload = 2; + string error = 3; +} + +message ClaimDeviceMsg { + int64 deviceIdMSB = 1; + int64 deviceIdLSB = 2; + string secretKey = 3; + int64 durationMs = 4; +} + +//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. +message SubscriptionInfoProto { + int64 lastActivityTime = 1; + bool attributeSubscription = 2; + bool rpcSubscription = 3; +} + +message SessionSubscriptionInfoProto { + SessionInfoProto sessionInfo = 1; + SubscriptionInfoProto subscriptionInfo = 2; +} + +message DeviceSessionsCacheEntry { + repeated SessionSubscriptionInfoProto sessions = 1; +} + +message TransportToDeviceActorMsg { + SessionInfoProto sessionInfo = 1; + SessionEventMsg sessionEvent = 2; + GetAttributeRequestMsg getAttributes = 3; + SubscribeToAttributeUpdatesMsg subscribeToAttributes = 4; + SubscribeToRPCMsg subscribeToRPC = 5; + ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 6; + SubscriptionInfoProto subscriptionInfo = 7; + ClaimDeviceMsg claimDevice = 8; +} + +message TransportToRuleEngineMsg { + SessionInfoProto sessionInfo = 1; + PostTelemetryMsg postTelemetry = 2; + PostAttributeMsg postAttributes = 3; + ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 4; + ToServerRpcRequestMsg toServerRPCCallRequest = 5; +} + +/** + * TB Core Data Structures + */ + +message TbSubscriptionProto { + string serviceId = 1; + string sessionId = 2; + int32 subscriptionId = 3; + string entityType = 4; + int64 tenantIdMSB = 5; + int64 tenantIdLSB = 6; + int64 entityIdMSB = 7; + int64 entityIdLSB = 8; +} + +message TbTimeSeriesSubscriptionProto { + TbSubscriptionProto sub = 1; + bool allKeys = 2; + repeated TbSubscriptionKetStateProto keyStates = 3; + int64 startTime = 4; + int64 endTime = 5; +} + +message TbAttributeSubscriptionProto { + TbSubscriptionProto sub = 1; + bool allKeys = 2; + repeated TbSubscriptionKetStateProto keyStates = 3; + string scope = 4; +} + +message TbSubscriptionUpdateProto { + string sessionId = 1; + int32 subscriptionId = 2; + int32 errorCode = 3; + string errorMsg = 4; + repeated TbSubscriptionUpdateValueListProto data = 5; +} + +message TbAttributeUpdateProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + string scope = 6; + repeated TsKvProto data = 7; +} + +message TbTimeSeriesUpdateProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + repeated TsKvProto data = 6; +} + +message TbSubscriptionCloseProto { + string sessionId = 1; + int32 subscriptionId = 2; +} + +message TbSubscriptionKetStateProto { + string key = 1; + int64 ts = 2; +} + +message TbSubscriptionUpdateValueListProto { + string key = 1; + repeated int64 ts = 2; + repeated string value = 3; +} + +/** + * TB Core to TB Core messages + */ + +message DeviceStateServiceMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; +} + +message SubscriptionMgrMsgProto { + TbTimeSeriesSubscriptionProto telemetrySub = 1; + TbAttributeSubscriptionProto attributeSub = 2; + TbSubscriptionCloseProto subClose = 3; + TbTimeSeriesUpdateProto tsUpdate = 4; + TbAttributeUpdateProto attrUpdate = 5; +} + +message LocalSubscriptionServiceMsgProto { + TbSubscriptionUpdateProto subUpdate = 1; +} + +message FromDeviceRPCResponseProto { + int64 requestIdMSB = 1; + int64 requestIdLSB = 2; + string response = 3; + int32 error = 4; +} +/** + * Main messages; + */ + +/* Request from Transport Service to ThingsBoard Core Service */ +message TransportApiRequestMsg { + ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; + ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; + GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; + GetTenantRoutingInfoRequestMsg getTenantRoutingInfoRequestMsg = 4; +} + +/* Response from ThingsBoard Core Service to Transport Service */ +message TransportApiResponseMsg { + ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; + GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; + GetTenantRoutingInfoResponseMsg getTenantRoutingInfoResponseMsg = 4; +} + +/* Messages that are handled by ThingsBoard Core Service */ +message ToCoreMsg { + TransportToDeviceActorMsg toDeviceActorMsg = 1; + DeviceStateServiceMsgProto deviceStateServiceMsg = 2; + SubscriptionMgrMsgProto toSubscriptionMgrMsg = 3; + bytes toDeviceActorNotificationMsg = 4; +} + +/* High priority messages with low latency are handled by ThingsBoard Core Service separately */ +message ToCoreNotificationMsg { + LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 1; + FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; + bytes componentLifecycleMsg = 3; +} + +/* Messages that are handled by ThingsBoard RuleEngine Service */ +message ToRuleEngineMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + bytes tbMsg = 3; + repeated string relationTypes = 4; + string failureMessage = 5; +} + +message ToRuleEngineNotificationMsg { + bytes componentLifecycleMsg = 1; + FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; +} + +/* Messages that are handled by ThingsBoard Transport Service */ +message ToTransportMsg { + int64 sessionIdMSB = 1; + int64 sessionIdLSB = 2; + SessionCloseNotificationProto sessionCloseNotification = 3; + GetAttributeResponseMsg getAttributesResponse = 4; + AttributeUpdateNotificationMsg attributeUpdateNotification = 5; + ToDeviceRpcRequestMsg toDeviceRequest = 6; + ToServerRpcResponseMsg toServerResponse = 7; +} diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java index 5be2f76258..864e7b2ab9 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java @@ -28,7 +28,7 @@ import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; * Created by ashvayka on 18.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") @Component public class CoapTransportContext extends TransportContext { diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index 9d2cff5c09..82678fde5d 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -303,6 +303,8 @@ public class CoapTransportResource extends CoapResource { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index d725d47a5f..fd4c8e2d95 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -20,14 +20,8 @@ import org.eclipse.californium.core.CoapResource; import org.eclipse.californium.core.CoapServer; import org.eclipse.californium.core.network.CoapEndpoint; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.transport.SessionMsgProcessor; -import org.thingsboard.server.common.transport.auth.DeviceAuthService; -import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -36,7 +30,7 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; @Service("CoapTransportService") -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") @Slf4j public class CoapTransportService { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index 9df658224a..c2f171e39b 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -61,7 +61,7 @@ import java.util.function.Consumer; * @author Andrew Shvayka */ @RestController -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") @RequestMapping("/api/v1") @Slf4j public class DeviceApiController { @@ -221,6 +221,8 @@ public class DeviceApiController { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java index 3d344d8dc5..28c38bdda6 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java @@ -29,7 +29,7 @@ import javax.annotation.PostConstruct; * Created by ashvayka on 04.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") @Component public class HttpTransportContext extends TransportContext { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java index 107639f302..b058a1c260 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java @@ -15,33 +15,23 @@ */ package org.thingsboard.server.transport.mqtt; -import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.handler.ssl.SslHandler; -import lombok.Data; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import org.thingsboard.server.common.transport.TransportContext; -import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - /** * Created by ashvayka on 04.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") @Component +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") public class MqttTransportContext extends TransportContext { @Getter diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 84c7598d9b..7295cb786b 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -44,7 +44,7 @@ import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.service.AbstractTransportService; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; @@ -410,13 +410,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private void processDisconnect(ChannelHandlerContext ctx) { ctx.close(); log.info("[{}] Client disconnected!", sessionId); - if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); - transportService.deregisterSession(sessionInfo); - if (gatewaySessionHandler != null) { - gatewaySessionHandler.onGatewayDisconnect(); - } - } + doDisconnect(); } private MqttConnAckMessage createMqttConnAckMsg(MqttConnectReturnCode returnCode) { @@ -485,9 +479,17 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void operationComplete(Future future) throws Exception { + doDisconnect(); + } + + private void doDisconnect() { if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); + transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); transportService.deregisterSession(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.onGatewayDisconnect(); + } + deviceSessionCtx.setDisconnected(); } } @@ -505,8 +507,10 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement .setDeviceIdLSB(msg.getDeviceInfo().getDeviceIdLSB()) .setTenantIdMSB(msg.getDeviceInfo().getTenantIdMSB()) .setTenantIdLSB(msg.getDeviceInfo().getTenantIdLSB()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.OPEN), null); + transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.OPEN), null); transportService.registerAsyncSession(sessionInfo, this); checkGatewaySession(); ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED)); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java index 8f752b5066..fb8a0b70be 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java @@ -36,7 +36,7 @@ import javax.annotation.PreDestroy; * @author Andrew Shvayka */ @Service("MqttTransportService") -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") @Slf4j public class MqttTransportService { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java index 299f5449d9..b4b5f98238 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java @@ -44,6 +44,8 @@ public class GatewayDeviceSessionCtx extends MqttDeviceAwareSessionContext imple .setDeviceIdLSB(deviceInfo.getDeviceIdLSB()) .setTenantIdMSB(deviceInfo.getTenantIdMSB()) .setTenantIdLSB(deviceInfo.getTenantIdLSB()) + .setDeviceName(deviceInfo.getDeviceName()) + .setDeviceType(deviceInfo.getDeviceType()) .build(); setDeviceInfo(deviceInfo); } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index e0a944ed60..de5f82f324 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.service.AbstractTransportService; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; @@ -121,7 +121,7 @@ public class GatewaySessionHandler { if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) { SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo(); transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx); - transportService.process(deviceSessionInfo, AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); + transportService.process(deviceSessionInfo, DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null); } @@ -376,7 +376,7 @@ public class GatewaySessionHandler { private void deregisterSession(String deviceName, GatewayDeviceSessionCtx deviceSessionCtx) { transportService.deregisterSession(deviceSessionCtx.getSessionInfo()); - transportService.process(deviceSessionCtx.getSessionInfo(), AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); + transportService.process(deviceSessionCtx.getSessionInfo(), DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); log.debug("[{}] Removed device [{}] from the gateway session", sessionId, deviceName); } diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 3920234f7c..af471e537c 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -36,6 +36,10 @@ + + org.thingsboard.common + queue + org.thingsboard.common data @@ -99,19 +103,6 @@ org.apache.commons commons-lang3 - - com.google.protobuf - protobuf-java - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - - diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java index ab67beee0b..c49b9c5bd3 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java @@ -20,7 +20,9 @@ import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -32,15 +34,16 @@ import java.util.concurrent.Executors; */ @Slf4j @Data -public class TransportContext { +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || '${service.type:null}'=='monolith'") +public abstract class TransportContext { protected final ObjectMapper mapper = new ObjectMapper(); @Autowired private TransportService transportService; - @Autowired - private TbNodeIdProvider nodeIdProvider; + private TbServiceInfoProvider serviceInfoProvider; @Getter private ExecutorService executor; @@ -58,7 +61,7 @@ public class TransportContext { } public String getNodeId() { - return nodeIdProvider.getNodeId(); + return serviceInfoProvider.getServiceId(); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index b0772de945..2af55c9206 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -15,15 +15,19 @@ */ package org.thingsboard.server.common.transport; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; @@ -35,14 +39,16 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce */ public interface TransportService { + GetTenantRoutingInfoResponseMsg getRoutingInfo(GetTenantRoutingInfoRequestMsg msg); + void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback); void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback); - void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, - TransportServiceCallback callback); + void process(GetOrCreateDeviceFromGatewayRequestMsg msg, + TransportServiceCallback callback); boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback); @@ -62,7 +68,7 @@ public interface TransportService { void process(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback); - void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback); + void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback); void process(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java deleted file mode 100644 index 77c45e1714..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java +++ /dev/null @@ -1,327 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.service; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.msg.tools.TbRateLimitsException; -import org.thingsboard.server.common.transport.SessionMsgListener; -import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.*; - -/** - * Created by ashvayka on 17.10.18. - */ -@Slf4j -public abstract class AbstractTransportService implements TransportService { - - @Value("${transport.rate_limits.enabled}") - private boolean rateLimitEnabled; - @Value("${transport.rate_limits.tenant}") - private String perTenantLimitsConf; - @Value("${transport.rate_limits.device}") - private String perDevicesLimitsConf; - @Value("${transport.sessions.inactivity_timeout}") - private long sessionInactivityTimeout; - @Value("${transport.sessions.report_timeout}") - private long sessionReportTimeout; - - protected ScheduledExecutorService schedulerExecutor; - protected ExecutorService transportCallbackExecutor; - - private ConcurrentMap sessions = new ConcurrentHashMap<>(); - - //TODO: Implement cleanup of this maps. - private ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); - private ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); - - @Override - public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { - sessions.putIfAbsent(toId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); - sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); - sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback)) { - reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); - } - } - - @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, - TransportServiceCallback callback) { - registerClaimingInfo(sessionInfo, msg, callback); - } - - @Override - public void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { - reportActivityInternal(sessionInfo); - } - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback); - - protected abstract void registerClaimingInfo(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback); - - private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { - UUID sessionId = toId(sessionInfo); - SessionMetaData sessionMetaData = sessions.get(sessionId); - if (sessionMetaData != null) { - sessionMetaData.updateLastActivityTime(); - } - return sessionMetaData; - } - - private void checkInactivityAndReportActivity() { - long expTime = System.currentTimeMillis() - sessionInactivityTimeout; - sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getLastActivityTime() < expTime) { - if (log.isDebugEnabled()) { - log.debug("[{}] Session has expired due to last activity time: {}", toId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); - } - process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); - sessions.remove(uuid); - sessionMD.getListener().onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); - } else { - if (sessionMD.getLastActivityTime() > sessionMD.getLastReportedActivityTime()) { - final long lastActivityTime = sessionMD.getLastActivityTime(); - process(sessionMD.getSessionInfo(), TransportProtos.SubscriptionInfoProto.newBuilder() - .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) - .setRpcSubscription(sessionMD.isSubscribedToRPC()) - .setLastActivityTime(sessionMD.getLastActivityTime()).build(), new TransportServiceCallback() { - @Override - public void onSuccess(Void msg) { - sessionMD.setLastReportedActivityTime(lastActivityTime); - } - - @Override - public void onError(Throwable e) { - log.warn("[{}] Failed to report last activity time", uuid, e); - } - }); - } - } - }); - } - - @Override - public void registerSyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout) { - SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); - sessions.putIfAbsent(toId(sessionInfo), currentSession); - - ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { - listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); - deregisterSession(sessionInfo); - }, timeout, TimeUnit.MILLISECONDS); - - currentSession.setScheduledFuture(executorFuture); - } - - @Override - public void deregisterSession(TransportProtos.SessionInfoProto sessionInfo) { - SessionMetaData currentSession = sessions.get(toId(sessionInfo)); - if (currentSession != null && currentSession.hasScheduledFuture()) { - log.debug("Stopping scheduler to avoid resending response if request has been ack."); - currentSession.getScheduledFuture().cancel(false); - } - sessions.remove(toId(sessionInfo)); - } - - @Override - public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { - if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); - } - if (!rateLimitEnabled) { - return true; - } - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(perTenantLimitsConf)); - if (!rateLimits.tryConsume()) { - if (callback != null) { - callback.onError(new TbRateLimitsException(EntityType.TENANT)); - } - if (log.isTraceEnabled()) { - log.trace("[{}][{}] Tenant level rate limit detected: {}", toId(sessionInfo), tenantId, msg); - } - return false; - } - DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - rateLimits = perDeviceLimits.computeIfAbsent(deviceId, id -> new TbRateLimits(perDevicesLimitsConf)); - if (!rateLimits.tryConsume()) { - if (callback != null) { - callback.onError(new TbRateLimitsException(EntityType.DEVICE)); - } - if (log.isTraceEnabled()) { - log.trace("[{}][{}] Device level rate limit detected: {}", toId(sessionInfo), deviceId, msg); - } - return false; - } - - return true; - } - - protected void processToTransportMsg(TransportProtos.DeviceActorToTransportMsg toSessionMsg) { - UUID sessionId = new UUID(toSessionMsg.getSessionIdMSB(), toSessionMsg.getSessionIdLSB()); - SessionMetaData md = sessions.get(sessionId); - if (md != null) { - SessionMsgListener listener = md.getListener(); - transportCallbackExecutor.submit(() -> { - if (toSessionMsg.hasGetAttributesResponse()) { - listener.onGetAttributesResponse(toSessionMsg.getGetAttributesResponse()); - } - if (toSessionMsg.hasAttributeUpdateNotification()) { - listener.onAttributeUpdate(toSessionMsg.getAttributeUpdateNotification()); - } - if (toSessionMsg.hasSessionCloseNotification()) { - listener.onRemoteSessionCloseCommand(toSessionMsg.getSessionCloseNotification()); - } - if (toSessionMsg.hasToDeviceRequest()) { - listener.onToDeviceRpcRequest(toSessionMsg.getToDeviceRequest()); - } - if (toSessionMsg.hasToServerResponse()) { - listener.onToServerRpcResponse(toSessionMsg.getToServerResponse()); - } - }); - if (md.getSessionType() == TransportProtos.SessionType.SYNC) { - deregisterSession(md.getSessionInfo()); - } - } else { - //TODO: should we notify the device actor about missed session? - log.debug("[{}] Missing session.", sessionId); - } - } - - protected UUID toId(TransportProtos.SessionInfoProto sessionInfo) { - return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); - } - - protected String getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { - return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()).toString(); - } - - public void init() { - if (rateLimitEnabled) { - //Just checking the configuration parameters - new TbRateLimits(perTenantLimitsConf); - new TbRateLimits(perDevicesLimitsConf); - } - this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); - this.transportCallbackExecutor = Executors.newWorkStealingPool(20); - this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); - } - - public void destroy() { - if (rateLimitEnabled) { - perTenantLimits.clear(); - perDeviceLimits.clear(); - } - if (schedulerExecutor != null) { - schedulerExecutor.shutdownNow(); - } - if (transportCallbackExecutor != null) { - transportCallbackExecutor.shutdownNow(); - } - } - - public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { - return TransportProtos.SessionEventMsg.newBuilder() - .setSessionType(TransportProtos.SessionType.ASYNC) - .setEvent(event).build(); - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java new file mode 100644 index 0000000000..c820167de1 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -0,0 +1,600 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.service; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.session.SessionMsgType; +import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.common.msg.tools.TbRateLimitsException; +import org.thingsboard.server.common.transport.SessionMsgListener; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.common.transport.util.JsonUtils; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.AsyncCallbackTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbTransportQueueFactory; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by ashvayka on 17.10.18. + */ +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport'") +public class DefaultTransportService implements TransportService { + + @Value("${transport.rate_limits.enabled}") + private boolean rateLimitEnabled; + @Value("${transport.rate_limits.tenant}") + private String perTenantLimitsConf; + @Value("${transport.rate_limits.device}") + private String perDevicesLimitsConf; + @Value("${transport.sessions.inactivity_timeout}") + private long sessionInactivityTimeout; + @Value("${transport.sessions.report_timeout}") + private long sessionReportTimeout; + @Value("${transport.client_side_rpc.timeout:60000}") + private long clientSideRpcTimeout; + @Value("${queue.transport.poll_interval}") + private int notificationsPollDuration; + + private final Gson gson = new Gson(); + private final TbTransportQueueFactory queueProvider; + private final TbQueueProducerProvider producerProvider; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; + protected TbQueueProducer> ruleEngineMsgProducer; + protected TbQueueProducer> tbCoreMsgProducer; + protected TbQueueConsumer> transportNotificationsConsumer; + + protected ScheduledExecutorService schedulerExecutor; + protected ExecutorService transportCallbackExecutor; + + private final ConcurrentMap sessions = new ConcurrentHashMap<>(); + private final Map toServerRpcPendingMap = new ConcurrentHashMap<>(); + //TODO: Implement cleanup of this maps. + private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); + + private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); + private volatile boolean stopped = false; + + public DefaultTransportService(TbServiceInfoProvider serviceInfoProvider, TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.queueProvider = queueProvider; + this.producerProvider = producerProvider; + this.partitionService = partitionService; + } + + @PostConstruct + public void init() { + if (rateLimitEnabled) { + //Just checking the configuration parameters + new TbRateLimits(perTenantLimitsConf); + new TbRateLimits(perDevicesLimitsConf); + } + this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); + this.transportCallbackExecutor = Executors.newWorkStealingPool(20); + this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); + transportApiRequestTemplate = queueProvider.createTransportApiRequestTemplate(); + ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); + tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); + transportNotificationsConsumer = queueProvider.createTransportNotificationsConsumer(); + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceInfoProvider.getServiceId()); + transportNotificationsConsumer.subscribe(Collections.singleton(tpi)); + transportApiRequestTemplate.init(); + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> records = transportNotificationsConsumer.poll(notificationsPollDuration); + if (records.size() == 0) { + continue; + } + records.forEach(record -> { + try { + processToTransportMsg(record.getValue()); + } catch (Throwable e) { + log.warn("Failed to process the notification.", e); + } + }); + transportNotificationsConsumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(notificationsPollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + } + }); + } + + @PreDestroy + public void destroy() { + if (rateLimitEnabled) { + perTenantLimits.clear(); + perDeviceLimits.clear(); + } + stopped = true; + + if (transportNotificationsConsumer != null) { + transportNotificationsConsumer.unsubscribe(); + } + if (schedulerExecutor != null) { + schedulerExecutor.shutdownNow(); + } + if (transportCallbackExecutor != null) { + transportCallbackExecutor.shutdownNow(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + if (transportApiRequestTemplate != null) { + transportApiRequestTemplate.stop(); + } + } + + @Override + public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { + sessions.putIfAbsent(toSessionId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); + } + + @Override + public TransportProtos.GetTenantRoutingInfoResponseMsg getRoutingInfo(TransportProtos.GetTenantRoutingInfoRequestMsg msg) { + TbProtoQueueMsg protoMsg = + new TbProtoQueueMsg<>(UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder().setGetTenantRoutingInfoRequestMsg(msg).build()); + try { + TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); + return response.getValue().getGetTenantRoutingInfoResponseMsg(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + @Override + public void process(TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { + if (log.isTraceEnabled()) { + log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); + } + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscriptionInfo(msg).build(), callback); + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSessionEvent(msg).build(), callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + MsgPackCallback packCallback = new MsgPackCallback(msg.getTsKvListCount(), callback); + for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) { + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + metaData.putValue("ts", tsKv.getTs() + ""); + JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, gson.toJson(json)); + sendToRuleEngine(tenantId, tbMsg, packCallback); + } + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, gson.toJson(json)); + sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setGetAttributes(msg).build(), callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); + sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToAttributes(msg).build(), callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); + sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToRPC(msg).build(), callback); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToDeviceRPCCallResponse(msg).build(), callback); + } + } + + private void processTimeout(String requestId) { + RpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); + if (data != null) { + SessionMetaData md = sessions.get(data.getSessionId()); + if (md != null) { + SessionMsgListener listener = md.getListener(); + transportCallbackExecutor.submit(() -> { + TransportProtos.ToServerRpcResponseMsg responseMsg = + TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(data.getRequestId()) + .setError("timeout").build(); + listener.onToServerRpcResponse(responseMsg); + }); + if (md.getSessionType() == TransportProtos.SessionType.SYNC) { + deregisterSession(md.getSessionInfo()); + } + } else { + log.debug("[{}] Missing session.", data.getSessionId()); + } + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); + UUID sessionId = toSessionId(sessionInfo); + TenantId tenantId = getTenantId(sessionInfo); + DeviceId deviceId = getDeviceId(sessionInfo); + JsonObject json = new JsonObject(); + json.addProperty("method", msg.getMethodName()); + json.add("params", JsonUtils.parse(msg.getParams())); + + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + metaData.putValue("requestId", Integer.toString(msg.getRequestId())); + metaData.putValue("serviceId", serviceInfoProvider.getServiceId()); + metaData.putValue("sessionId", sessionId.toString()); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json)); + sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); + + String requestId = sessionId + "-" + msg.getRequestId(); + toServerRpcPendingMap.put(requestId, new RpcRequestMetadata(sessionId, msg.getRequestId())); + schedulerExecutor.schedule(() -> processTimeout(requestId), clientSideRpcTimeout, TimeUnit.MILLISECONDS); + } + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setClaimDevice(msg).build(), callback); + } + } + + @Override + public void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { + reportActivityInternal(sessionInfo); + } + + private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { + UUID sessionId = toSessionId(sessionInfo); + SessionMetaData sessionMetaData = sessions.get(sessionId); + if (sessionMetaData != null) { + sessionMetaData.updateLastActivityTime(); + } + return sessionMetaData; + } + + private void checkInactivityAndReportActivity() { + long expTime = System.currentTimeMillis() - sessionInactivityTimeout; + sessions.forEach((uuid, sessionMD) -> { + if (sessionMD.getLastActivityTime() < expTime) { + if (log.isDebugEnabled()) { + log.debug("[{}] Session has expired due to last activity time: {}", toSessionId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); + } + process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); + sessions.remove(uuid); + sessionMD.getListener().onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); + } else { + if (sessionMD.getLastActivityTime() > sessionMD.getLastReportedActivityTime()) { + final long lastActivityTime = sessionMD.getLastActivityTime(); + process(sessionMD.getSessionInfo(), TransportProtos.SubscriptionInfoProto.newBuilder() + .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) + .setRpcSubscription(sessionMD.isSubscribedToRPC()) + .setLastActivityTime(sessionMD.getLastActivityTime()).build(), new TransportServiceCallback() { + @Override + public void onSuccess(Void msg) { + sessionMD.setLastReportedActivityTime(lastActivityTime); + } + + @Override + public void onError(Throwable e) { + log.warn("[{}] Failed to report last activity time", uuid, e); + } + }); + } + } + }); + } + + @Override + public void registerSyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout) { + SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); + sessions.putIfAbsent(toSessionId(sessionInfo), currentSession); + + ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { + listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); + deregisterSession(sessionInfo); + }, timeout, TimeUnit.MILLISECONDS); + + currentSession.setScheduledFuture(executorFuture); + } + + @Override + public void deregisterSession(TransportProtos.SessionInfoProto sessionInfo) { + SessionMetaData currentSession = sessions.get(toSessionId(sessionInfo)); + if (currentSession != null && currentSession.hasScheduledFuture()) { + log.debug("Stopping scheduler to avoid resending response if request has been ack."); + currentSession.getScheduledFuture().cancel(false); + } + sessions.remove(toSessionId(sessionInfo)); + } + + @Override + public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { + if (log.isTraceEnabled()) { + log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); + } + if (!rateLimitEnabled) { + return true; + } + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(perTenantLimitsConf)); + if (!rateLimits.tryConsume()) { + if (callback != null) { + callback.onError(new TbRateLimitsException(EntityType.TENANT)); + } + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Tenant level rate limit detected: {}", toSessionId(sessionInfo), tenantId, msg); + } + return false; + } + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + rateLimits = perDeviceLimits.computeIfAbsent(deviceId, id -> new TbRateLimits(perDevicesLimitsConf)); + if (!rateLimits.tryConsume()) { + if (callback != null) { + callback.onError(new TbRateLimitsException(EntityType.DEVICE)); + } + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Device level rate limit detected: {}", toSessionId(sessionInfo), deviceId, msg); + } + return false; + } + + return true; + } + + protected void processToTransportMsg(TransportProtos.ToTransportMsg toSessionMsg) { + UUID sessionId = new UUID(toSessionMsg.getSessionIdMSB(), toSessionMsg.getSessionIdLSB()); + SessionMetaData md = sessions.get(sessionId); + if (md != null) { + SessionMsgListener listener = md.getListener(); + transportCallbackExecutor.submit(() -> { + if (toSessionMsg.hasGetAttributesResponse()) { + listener.onGetAttributesResponse(toSessionMsg.getGetAttributesResponse()); + } + if (toSessionMsg.hasAttributeUpdateNotification()) { + listener.onAttributeUpdate(toSessionMsg.getAttributeUpdateNotification()); + } + if (toSessionMsg.hasSessionCloseNotification()) { + listener.onRemoteSessionCloseCommand(toSessionMsg.getSessionCloseNotification()); + } + if (toSessionMsg.hasToDeviceRequest()) { + listener.onToDeviceRpcRequest(toSessionMsg.getToDeviceRequest()); + } + if (toSessionMsg.hasToServerResponse()) { + String requestId = sessionId + "-" + toSessionMsg.getToServerResponse().getRequestId(); + toServerRpcPendingMap.remove(requestId); + listener.onToServerRpcResponse(toSessionMsg.getToServerResponse()); + } + }); + if (md.getSessionType() == TransportProtos.SessionType.SYNC) { + deregisterSession(md.getSessionInfo()); + } + } else { + //TODO: should we notify the device actor about missed session? + log.debug("[{}] Missing session.", sessionId); + } + } + + protected UUID toSessionId(TransportProtos.SessionInfoProto sessionInfo) { + return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); + } + + protected UUID getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { + return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()); + } + + protected TenantId getTenantId(TransportProtos.SessionInfoProto sessionInfo) { + return new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + } + + protected DeviceId getDeviceId(TransportProtos.SessionInfoProto sessionInfo) { + return new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + } + + public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { + return TransportProtos.SessionEventMsg.newBuilder() + .setSessionType(TransportProtos.SessionType.ASYNC) + .setEvent(event).build(); + } + + protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); + tbCoreMsgProducer.send(tpi, + new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? + new TransportTbQueueCallback(callback) : null); + } + + protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); + ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); + } + + private class TransportTbQueueCallback implements TbQueueCallback { + private final TransportServiceCallback callback; + + private TransportTbQueueCallback(TransportServiceCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null)); + } + + @Override + public void onFailure(Throwable t) { + DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onError(t)); + } + } + + private class MsgPackCallback implements TbQueueCallback { + private final AtomicInteger msgCount; + private final TransportServiceCallback callback; + + public MsgPackCallback(Integer msgCount, TransportServiceCallback callback) { + this.msgCount = new AtomicInteger(msgCount); + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (msgCount.decrementAndGet() <= 0) { + DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null)); + } + } + + @Override + public void onFailure(Throwable t) { + callback.onError(t); + } + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java deleted file mode 100644 index 17031791d5..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java +++ /dev/null @@ -1,339 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.service; - -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.kafka.AsyncCallbackTemplate; -import org.thingsboard.server.kafka.TBKafkaAdmin; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaRequestTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.time.Duration; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * Created by ashvayka on 05.10.18. - */ -@ConditionalOnExpression("'${transport.type:null}'=='null'") -@Service -@Slf4j -public class RemoteTransportService extends AbstractTransportService { - - @Value("${kafka.rule_engine.topic}") - private String ruleEngineTopic; - @Value("${kafka.notifications.topic}") - private String notificationsTopic; - @Value("${kafka.notifications.poll_interval}") - private int notificationsPollDuration; - @Value("${kafka.notifications.auto_commit_interval}") - private int notificationsAutoCommitInterval; - @Value("${kafka.transport_api.requests_topic}") - private String transportApiRequestsTopic; - @Value("${kafka.transport_api.responses_topic}") - private String transportApiResponsesTopic; - @Value("${kafka.transport_api.max_pending_requests}") - private long maxPendingRequests; - @Value("${kafka.transport_api.max_requests_timeout}") - private long maxRequestsTimeout; - @Value("${kafka.transport_api.response_poll_interval}") - private int responsePollDuration; - @Value("${kafka.transport_api.response_auto_commit_interval}") - private int autoCommitInterval; - - @Autowired - private TbKafkaSettings kafkaSettings; - //We use this to get the node id. We should replace this with a component that provides the node id. - @Autowired - private TbNodeIdProvider nodeIdProvider; - - private TbKafkaRequestTemplate transportApiTemplate; - private TBKafkaProducerTemplate ruleEngineProducer; - private TBKafkaConsumerTemplate mainConsumer; - - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("remote-transport-consumer")); - - private volatile boolean stopped = false; - - @PostConstruct - public void init() { - super.init(); - - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-api-request-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(transportApiRequestsTopic); - requestBuilder.encoder(new TransportApiRequestEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportApiResponsesTopic + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("transport-api-client-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("transport-api-client"); - responseBuilder.autoCommit(true); - responseBuilder.autoCommitIntervalMs(autoCommitInterval); - responseBuilder.decoder(new TransportApiResponseDecoder()); - - TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder - builder = TbKafkaRequestTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(maxPendingRequests); - builder.maxRequestTimeout(maxRequestsTimeout); - builder.pollInterval(responsePollDuration); - transportApiTemplate = builder.build(); - transportApiTemplate.init(); - - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder ruleEngineProducerBuilder = TBKafkaProducerTemplate.builder(); - ruleEngineProducerBuilder.settings(kafkaSettings); - ruleEngineProducerBuilder.clientId("producer-rule-engine-request-" + nodeIdProvider.getNodeId()); - ruleEngineProducerBuilder.defaultTopic(ruleEngineTopic); - ruleEngineProducerBuilder.encoder(new ToRuleEngineMsgEncoder()); - ruleEngineProducer = ruleEngineProducerBuilder.build(); - ruleEngineProducer.init(); - - String notificationsTopicName = notificationsTopic + "." + nodeIdProvider.getNodeId(); - - try { - TBKafkaAdmin admin = new TBKafkaAdmin(kafkaSettings); - CreateTopicsResult result = admin.createTopic(new NewTopic(notificationsTopicName, 1, (short) 1)); - result.all().get(); - } catch (Exception e) { - log.trace("Failed to create topic: {}", e.getMessage(), e); - } - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder mainConsumerBuilder = TBKafkaConsumerTemplate.builder(); - mainConsumerBuilder.settings(kafkaSettings); - mainConsumerBuilder.topic(notificationsTopicName); - mainConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); - mainConsumerBuilder.groupId("transport"); - mainConsumerBuilder.autoCommit(true); - mainConsumerBuilder.autoCommitIntervalMs(notificationsAutoCommitInterval); - mainConsumerBuilder.decoder(new ToTransportMsgResponseDecoder()); - mainConsumer = mainConsumerBuilder.build(); - mainConsumer.subscribe(); - - mainConsumerExecutor.execute(() -> { - while (!stopped) { - try { - ConsumerRecords records = mainConsumer.poll(Duration.ofMillis(notificationsPollDuration)); - records.forEach(record -> { - try { - ToTransportMsg toTransportMsg = mainConsumer.decode(record); - if (toTransportMsg.hasToDeviceSessionMsg()) { - processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); - } - } catch (Throwable e) { - log.warn("Failed to process the notification.", e); - } - }); - } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(notificationsPollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - @PreDestroy - public void destroy() { - super.destroy(); - stopped = true; - if (transportApiTemplate != null) { - transportApiTemplate.stop(); - } - if (mainConsumer != null) { - mainConsumer.unsubscribe(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - } - - @Override - public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getToken(), - TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), - response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getHash(), - TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), - response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getDeviceName(), - TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), - response -> callback.onSuccess(response.getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback) { - if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); - } - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscriptionInfo(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSessionEvent(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostTelemetry(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setGetAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToRPC(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToDeviceRPCCallResponse(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToServerRPCCallRequest(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setClaimDevice(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - private void send(SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - ruleEngineProducer.send(getRoutingKey(sessionInfo), toRuleEngineMsg, (metadata, exception) -> { - if (callback != null) { - if (exception == null) { - this.transportCallbackExecutor.submit(() -> { - callback.onSuccess(null); - }); - } else { - this.transportCallbackExecutor.submit(() -> { - callback.onError(exception); - }); - } - } - }); - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java similarity index 76% rename from common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java index d06046fd71..2c5da0d3ea 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java @@ -13,20 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.core; +package org.thingsboard.server.common.transport.service; import lombok.Data; -/** - * @author Andrew Shvayka - */ -@Data -public class ToServerRpcResponseMsg { +import java.util.UUID; +@Data +public class RpcRequestMetadata { + private final UUID sessionId; private final int requestId; - private final String data; - - public boolean isSuccess() { - return true; - } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java index 42a65440a0..fc1445539b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java index c07e406d2e..cc154f87b2 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java @@ -15,8 +15,9 @@ */ package org.thingsboard.server.common.transport.service; +import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; @@ -24,8 +25,9 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class ToTransportMsgResponseDecoder implements TbKafkaDecoder { + @Override - public ToTransportMsg decode(byte[] data) throws IOException { - return ToTransportMsg.parseFrom(data); + public ToTransportMsg decode(TbQueueMsg msg) throws IOException { + return ToTransportMsg.parseFrom(msg.getData()); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java index 6c4b362f03..3971fdbf83 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java index a62e696500..5fdc58c068 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java @@ -15,8 +15,9 @@ */ package org.thingsboard.server.common.transport.service; +import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; @@ -24,8 +25,9 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class TransportApiResponseDecoder implements TbKafkaDecoder { + @Override - public TransportApiResponseMsg decode(byte[] data) throws IOException { - return TransportApiResponseMsg.parseFrom(data); + public TransportApiResponseMsg decode(TbQueueMsg msg) throws IOException { + return TransportApiResponseMsg.parseFrom(msg.getData()); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java new file mode 100644 index 0000000000..f2534a64c9 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") +public class TransportTenantRoutingInfoService implements TenantRoutingInfoService { + + private TransportService transportService; + + @Lazy + @Autowired + public void setTransportService(TransportService transportService) { + this.transportService = transportService; + } + + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + GetTenantRoutingInfoRequestMsg msg = GetTenantRoutingInfoRequestMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .build(); + GetTenantRoutingInfoResponseMsg routingInfo = transportService.getRoutingInfo(msg); + return new TenantRoutingInfo(tenantId, routingInfo.getIsolatedTbCore(), routingInfo.getIsolatedTbRuleEngine()); + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java index 7c590ec925..c312b3a4d9 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java @@ -35,6 +35,7 @@ public abstract class DeviceAwareSessionContext implements SessionContext { private volatile DeviceId deviceId; @Getter private volatile DeviceInfoProto deviceInfo; + private volatile boolean connected; public DeviceId getDeviceId() { return deviceId; @@ -42,10 +43,15 @@ public abstract class DeviceAwareSessionContext implements SessionContext { public void setDeviceInfo(DeviceInfoProto deviceInfo) { this.deviceInfo = deviceInfo; + this.connected = true; this.deviceId = new DeviceId(new UUID(deviceInfo.getDeviceIdMSB(), deviceInfo.getDeviceIdLSB())); } public boolean isConnected() { - return deviceInfo != null; + return connected; + } + + public void setDisconnected() { + this.connected = false; } } diff --git a/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/utils/JsonUtils.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java index 5621362c09..583b73dba1 100644 --- a/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.transport.util; import com.google.gson.JsonElement; import com.google.gson.JsonObject; diff --git a/common/transport/transport-api/src/main/proto/transport.proto b/common/transport/transport-api/src/main/proto/transport.proto deleted file mode 100644 index e8b513574a..0000000000 --- a/common/transport/transport-api/src/main/proto/transport.proto +++ /dev/null @@ -1,244 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; -package transport; - -option java_package = "org.thingsboard.server.gen.transport"; -option java_outer_classname = "TransportProtos"; - -/** - * Data Structures; - */ -message SessionInfoProto { - string nodeId = 1; - int64 sessionIdMSB = 2; - int64 sessionIdLSB = 3; - int64 tenantIdMSB = 4; - int64 tenantIdLSB = 5; - int64 deviceIdMSB = 6; - int64 deviceIdLSB = 7; -} - -enum SessionEvent { - OPEN = 0; - CLOSED = 1; -} - -enum SessionType { - SYNC = 0; - ASYNC = 1; -} - -enum KeyValueType { - BOOLEAN_V = 0; - LONG_V = 1; - DOUBLE_V = 2; - STRING_V = 3; - JSON_V = 4; -} - -message KeyValueProto { - string key = 1; - KeyValueType type = 2; - bool bool_v = 3; - int64 long_v = 4; - double double_v = 5; - string string_v = 6; - string json_v = 7; -} - -message TsKvProto { - int64 ts = 1; - KeyValueProto kv = 2; -} - -message TsKvListProto { - int64 ts = 1; - repeated KeyValueProto kv = 2; -} - -message DeviceInfoProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 deviceIdMSB = 3; - int64 deviceIdLSB = 4; - string deviceName = 5; - string deviceType = 6; - string additionalInfo = 7; -} - -/** - * Messages that use Data Structures; - */ -message SessionEventMsg { - SessionType sessionType = 1; - SessionEvent event = 2; -} - -message PostTelemetryMsg { - repeated TsKvListProto tsKvList = 1; -} - -message PostAttributeMsg { - repeated KeyValueProto kv = 1; -} - -message GetAttributeRequestMsg { - int32 requestId = 1; - repeated string clientAttributeNames = 2; - repeated string sharedAttributeNames = 3; -} - -message GetAttributeResponseMsg { - int32 requestId = 1; - repeated TsKvProto clientAttributeList = 2; - repeated TsKvProto sharedAttributeList = 3; - repeated string deletedAttributeKeys = 4; - string error = 5; -} - -message AttributeUpdateNotificationMsg { - repeated TsKvProto sharedUpdated = 1; - repeated string sharedDeleted = 2; -} - -message ValidateDeviceTokenRequestMsg { - string token = 1; -} - -message ValidateDeviceX509CertRequestMsg { - string hash = 1; -} - -message ValidateDeviceCredentialsResponseMsg { - DeviceInfoProto deviceInfo = 1; - string credentialsBody = 2; -} - -message GetOrCreateDeviceFromGatewayRequestMsg { - int64 gatewayIdMSB = 1; - int64 gatewayIdLSB = 2; - string deviceName = 3; - string deviceType = 4; -} - -message GetOrCreateDeviceFromGatewayResponseMsg { - DeviceInfoProto deviceInfo = 1; -} - -message SessionCloseNotificationProto { - string message = 1; -} - -message SubscribeToAttributeUpdatesMsg { - bool unsubscribe = 1; -} - -message SubscribeToRPCMsg { - bool unsubscribe = 1; -} - -message ToDeviceRpcRequestMsg { - int32 requestId = 1; - string methodName = 2; - string params = 3; -} - -message ToDeviceRpcResponseMsg { - int32 requestId = 1; - string payload = 2; -} - -message ToServerRpcRequestMsg { - int32 requestId = 1; - string methodName = 2; - string params = 3; -} - -message ToServerRpcResponseMsg { - int32 requestId = 1; - string payload = 2; - string error = 3; -} - -message ClaimDeviceMsg { - int64 deviceIdMSB = 1; - int64 deviceIdLSB = 2; - string secretKey = 3; - int64 durationMs = 4; -} - -//Used to report session state to tb-node and persist this state in the cache on the tb-node level. -message SubscriptionInfoProto { - int64 lastActivityTime = 1; - bool attributeSubscription = 2; - bool rpcSubscription = 3; -} - -message SessionSubscriptionInfoProto { - SessionInfoProto sessionInfo = 1; - SubscriptionInfoProto subscriptionInfo = 2; -} - -message DeviceSessionsCacheEntry { - repeated SessionSubscriptionInfoProto sessions = 1; -} - -message TransportToDeviceActorMsg { - SessionInfoProto sessionInfo = 1; - SessionEventMsg sessionEvent = 2; - PostTelemetryMsg postTelemetry = 3; - PostAttributeMsg postAttributes = 4; - GetAttributeRequestMsg getAttributes = 5; - SubscribeToAttributeUpdatesMsg subscribeToAttributes = 6; - SubscribeToRPCMsg subscribeToRPC = 7; - ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 8; - ToServerRpcRequestMsg toServerRPCCallRequest = 9; - SubscriptionInfoProto subscriptionInfo = 10; - ClaimDeviceMsg claimDevice = 11; -} - -message DeviceActorToTransportMsg { - int64 sessionIdMSB = 1; - int64 sessionIdLSB = 2; - SessionCloseNotificationProto sessionCloseNotification = 3; - GetAttributeResponseMsg getAttributesResponse = 4; - AttributeUpdateNotificationMsg attributeUpdateNotification = 5; - ToDeviceRpcRequestMsg toDeviceRequest = 6; - ToServerRpcResponseMsg toServerResponse = 7; -} - -/** - * Main messages; - */ -message ToRuleEngineMsg { - TransportToDeviceActorMsg toDeviceActorMsg = 1; -} - -message ToTransportMsg { - DeviceActorToTransportMsg toDeviceSessionMsg = 1; -} - -message TransportApiRequestMsg { - ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; - ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; - GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; -} - -message TransportApiResponseMsg { - ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; - GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 453f1ea0be..41f8a2b630 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -29,7 +29,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java b/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java index 664d7b382f..68e398131b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java @@ -151,12 +151,12 @@ public class CassandraBaseComponentDescriptorDao extends CassandraAbstractSearch } private Optional saveIfNotExist(TenantId tenantId, ComponentDescriptorEntity entity) { - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } ResultSet rs = executeRead(tenantId, QueryBuilder.insertInto(getColumnFamilyName()) - .value(ModelConstants.ID_PROPERTY, entity.getId()) + .value(ModelConstants.ID_PROPERTY, entity.getUuid()) .value(ModelConstants.COMPONENT_DESCRIPTOR_NAME_PROPERTY, entity.getName()) .value(ModelConstants.COMPONENT_DESCRIPTOR_CLASS_PROPERTY, entity.getClazz()) .value(ModelConstants.COMPONENT_DESCRIPTOR_TYPE_PROPERTY, entity.getType()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 5867e505b6..7080722056 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -23,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java index bdd0201aa5..f2660e512d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java @@ -184,11 +184,11 @@ public class CassandraBaseEventDao extends CassandraAbstractSearchTimeDao> saveAsync(TenantId tenantId, EventEntity entity, boolean ifNotExists, int ttl) { - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } Insert insert = QueryBuilder.insertInto(getColumnFamilyName()) - .value(ModelConstants.ID_PROPERTY, entity.getId()) + .value(ModelConstants.ID_PROPERTY, entity.getUuid()) .value(ModelConstants.EVENT_TENANT_ID_PROPERTY, entity.getTenantId()) .value(ModelConstants.EVENT_ENTITY_TYPE_PROPERTY, entity.getEntityType()) .value(ModelConstants.EVENT_ENTITY_ID_PROPERTY, entity.getEntityId()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java index 386220df55..b5bda06a47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java @@ -19,8 +19,8 @@ import java.util.UUID; public interface BaseEntity extends ToData { - UUID getId(); + UUID getUuid(); - void setId(UUID id); + void setUuid(UUID id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java index d75262e311..44fd70f977 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java @@ -35,14 +35,15 @@ public abstract class BaseSqlEntity implements BaseEntity { protected String id; @Override - public UUID getId() { + public UUID getUuid() { if (id == null) { return null; } return UUIDConverter.fromString(id); } - public void setId(UUID id) { + @Override + public void setUuid(UUID id) { this.id = UUIDConverter.fromTimeUUID(id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 96ce14c459..2db9251e49 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -112,6 +112,8 @@ public class ModelConstants { public static final String TENANT_TITLE_PROPERTY = TITLE_PROPERTY; public static final String TENANT_REGION_PROPERTY = "region"; public static final String TENANT_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY; + public static final String TENANT_ISOLATED_TB_CORE = "isolated_tb_core"; + public static final String TENANT_ISOLATED_TB_RULE_ENGINE = "isolated_tb_rule_engine"; public static final String TENANT_BY_REGION_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "tenant_by_region_and_search_text"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java index 93cc934a06..5d8f5d671f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java @@ -61,11 +61,11 @@ public final class AdminSettingsEntity implements BaseEntity { this.jsonValue = adminSettings.getJsonValue(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java index 0cc8dfab72..819daa800b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java @@ -27,7 +27,7 @@ import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -141,11 +141,11 @@ public final class AlarmEntity implements BaseEntity { } } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java index 2554e0c3ec..57ec0ee511 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java @@ -94,11 +94,11 @@ public final class AssetEntity implements SearchTextEntity { this.additionalInfo = asset.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java index 04e77aad8a..b406a6b341 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java @@ -94,12 +94,12 @@ public class AuditLogEntity implements BaseEntity { private String actionFailureDetails; @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java index 9a6827aa05..1af3b8a8fa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java @@ -98,12 +98,12 @@ public class ComponentDescriptorEntity implements SearchTextEntity { this.additionalInfo = customer.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java index 9fe999a0e5..a48305f630 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java @@ -98,11 +98,11 @@ public final class DashboardEntity implements SearchTextEntity { this.configuration = dashboard.getConfiguration(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java index b54252fdfc..69eb43744c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java @@ -91,11 +91,11 @@ public class DashboardInfoEntity implements SearchTextEntity { } } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java index 6900014d21..316386f7c9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java @@ -74,11 +74,11 @@ public final class DeviceCredentialsEntity implements BaseEntity { this.additionalInfo = device.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java index 80bc9a3276..5cf13665b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java @@ -161,4 +161,14 @@ public class EntityViewEntity implements SearchTextEntity { entityView.setAdditionalInfo(additionalInfo); return entityView; } + + @Override + public UUID getUuid() { + return getId(); + } + + @Override + public void setUuid(UUID id) { + this.id = id; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java index 2645d29df3..8c753a743a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java @@ -94,12 +94,12 @@ public class EventEntity implements BaseEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java index bb6a194801..7021ff708e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java @@ -102,12 +102,12 @@ public class RuleChainEntity implements SearchTextEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java index e2dc2fa244..9a086fe568 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java @@ -96,12 +96,12 @@ public class RuleNodeEntity implements SearchTextEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java index 0ce7990755..cae31c2a31 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java @@ -24,6 +24,7 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.SearchTextEntity; import org.thingsboard.server.dao.model.type.JsonCodec; @@ -55,16 +56,16 @@ public final class TenantEntity implements SearchTextEntity { @Column(name = TENANT_TITLE_PROPERTY) private String title; - + @Column(name = SEARCH_TEXT_PROPERTY) private String searchText; @Column(name = TENANT_REGION_PROPERTY) private String region; - + @Column(name = COUNTRY_PROPERTY) private String country; - + @Column(name = STATE_PROPERTY) private String state; @@ -89,6 +90,12 @@ public final class TenantEntity implements SearchTextEntity { @Column(name = TENANT_ADDITIONAL_INFO_PROPERTY, codec = JsonCodec.class) private JsonNode additionalInfo; + @Column(name = ModelConstants.TENANT_ISOLATED_TB_CORE) + private boolean isolatedTbCore; + + @Column(name = ModelConstants.TENANT_ISOLATED_TB_RULE_ENGINE) + private boolean isolatedTbRuleEngine; + public TenantEntity() { super(); } @@ -108,13 +115,15 @@ public final class TenantEntity implements SearchTextEntity { this.phone = tenant.getPhone(); this.email = tenant.getEmail(); this.additionalInfo = tenant.getAdditionalInfo(); + this.isolatedTbCore = tenant.isIsolatedTbCore(); + this.isolatedTbRuleEngine = tenant.isIsolatedTbRuleEngine(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } @@ -206,6 +215,22 @@ public final class TenantEntity implements SearchTextEntity { this.additionalInfo = additionalInfo; } + public boolean isIsolatedTbCore() { + return isolatedTbCore; + } + + public void setIsolatedTbCore(boolean isolatedTbCore) { + this.isolatedTbCore = isolatedTbCore; + } + + public boolean isIsolatedTbRuleEngine() { + return isolatedTbRuleEngine; + } + + public void setIsolatedTbRuleEngine(boolean isolatedTbRuleEngine) { + this.isolatedTbRuleEngine = isolatedTbRuleEngine; + } + @Override public String getSearchTextSource() { return getTitle(); @@ -215,7 +240,7 @@ public final class TenantEntity implements SearchTextEntity { public void setSearchText(String searchText) { this.searchText = searchText; } - + public String getSearchText() { return searchText; } @@ -235,6 +260,8 @@ public final class TenantEntity implements SearchTextEntity { tenant.setPhone(phone); tenant.setEmail(email); tenant.setAdditionalInfo(additionalInfo); + tenant.setIsolatedTbCore(isolatedTbCore); + tenant.setIsolatedTbRuleEngine(isolatedTbRuleEngine); return tenant; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java index 50b54bd8c9..6854d61f39 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java @@ -75,11 +75,11 @@ public final class UserCredentialsEntity implements BaseEntity this.resetToken = userCredentials.getResetToken(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java index d8399fc2b5..0c7625da78 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java @@ -101,11 +101,11 @@ public final class UserEntity implements SearchTextEntity { this.additionalInfo = user.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java index 65bc19f421..24ac3980cf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java @@ -82,12 +82,12 @@ public final class WidgetTypeEntity implements BaseEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java index b70a0f095b..8c11c05685 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java @@ -82,12 +82,12 @@ public final class WidgetsBundleEntity implements SearchTextEntity impl public AdminSettingsEntity(AdminSettings adminSettings) { if (adminSettings.getId() != null) { - this.setId(adminSettings.getId().getId()); + this.setUuid(adminSettings.getId().getId()); } this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java index 14adcdcd9c..8d29800022 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java @@ -26,7 +26,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -114,7 +114,7 @@ public final class AlarmEntity extends BaseSqlEntity implements BaseEntit public AlarmEntity(Alarm alarm) { if (alarm.getId() != null) { - this.setId(alarm.getId().getId()); + this.setUuid(alarm.getId().getId()); } if (alarm.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(alarm.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java index 0471d4a97d..6eff4cd521 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java @@ -78,7 +78,7 @@ public final class AssetEntity extends BaseSqlEntity implements SearchTex public AssetEntity(Asset asset) { if (asset.getId() != null) { - this.setId(asset.getId().getId()); + this.setUuid(asset.getId().getId()); } if (asset.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(asset.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java index 4de738ac73..e2573ea0d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java @@ -103,7 +103,7 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit public AuditLogEntity(AuditLog auditLog) { if (auditLog.getId() != null) { - this.setId(auditLog.getId().getId()); + this.setUuid(auditLog.getId().getId()); } if (auditLog.getTenantId() != null) { this.tenantId = toString(auditLog.getTenantId().getId()); @@ -128,8 +128,8 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit @Override public AuditLog toData() { - AuditLog auditLog = new AuditLog(new AuditLogId(getId())); - auditLog.setCreatedTime(UUIDs.unixTimestamp(getId())); + AuditLog auditLog = new AuditLog(new AuditLogId(this.getUuid())); + auditLog.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { auditLog.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java index 26946cf207..d5b287fe9b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java @@ -34,7 +34,6 @@ import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Table; -import javax.persistence.UniqueConstraint; @Data @EqualsAndHashCode(callSuper = true) @@ -72,7 +71,7 @@ public class ComponentDescriptorEntity extends BaseSqlEntity implements Sea public CustomerEntity(Customer customer) { if (customer.getId() != null) { - this.setId(customer.getId().getId()); + this.setUuid(customer.getId().getId()); } this.tenantId = UUIDConverter.fromTimeUUID(customer.getTenantId().getId()); this.title = customer.getTitle(); @@ -111,8 +111,8 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea @Override public Customer toData() { - Customer customer = new Customer(new CustomerId(getId())); - customer.setCreatedTime(UUIDs.unixTimestamp(getId())); + Customer customer = new Customer(new CustomerId(this.getUuid())); + customer.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); customer.setTenantId(new TenantId(UUIDConverter.fromString(tenantId))); customer.setTitle(title); customer.setCountry(country); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java index b606c35d4e..07cec4d12f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java @@ -75,7 +75,7 @@ public final class DashboardEntity extends BaseSqlEntity implements S public DashboardEntity(Dashboard dashboard) { if (dashboard.getId() != null) { - this.setId(dashboard.getId().getId()); + this.setUuid(dashboard.getId().getId()); } if (dashboard.getTenantId() != null) { this.tenantId = toString(dashboard.getTenantId().getId()); @@ -103,8 +103,8 @@ public final class DashboardEntity extends BaseSqlEntity implements S @Override public Dashboard toData() { - Dashboard dashboard = new Dashboard(new DashboardId(this.getId())); - dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getId())); + Dashboard dashboard = new Dashboard(new DashboardId(this.getUuid())); + dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { dashboard.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java index b1ecb2572b..f26cf21934 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java @@ -66,7 +66,7 @@ public class DashboardInfoEntity extends BaseSqlEntity implements public DashboardInfoEntity(DashboardInfo dashboardInfo) { if (dashboardInfo.getId() != null) { - this.setId(dashboardInfo.getId().getId()); + this.setUuid(dashboardInfo.getId().getId()); } if (dashboardInfo.getTenantId() != null) { this.tenantId = toString(dashboardInfo.getTenantId().getId()); @@ -97,8 +97,8 @@ public class DashboardInfoEntity extends BaseSqlEntity implements @Override public DashboardInfo toData() { - DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(getId())); - dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(getId())); + DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(this.getUuid())); + dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { dashboardInfo.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java index 1d47c7d63c..ba9d7ce637 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java @@ -57,7 +57,7 @@ public final class DeviceCredentialsEntity extends BaseSqlEntity implements SearchT public DeviceEntity(Device device) { if (device.getId() != null) { - this.setId(device.getId().getId()); + this.setUuid(device.getId().getId()); } if (device.getTenantId() != null) { this.tenantId = toString(device.getTenantId().getId()); @@ -95,8 +95,8 @@ public final class DeviceEntity extends BaseSqlEntity implements SearchT @Override public Device toData() { - Device device = new Device(new DeviceId(getId())); - device.setCreatedTime(UUIDs.unixTimestamp(getId())); + Device device = new Device(new DeviceId(this.getUuid())); + device.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { device.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java index 6a755d2803..fd446cc6ef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java @@ -99,7 +99,7 @@ public class EntityViewEntity extends BaseSqlEntity implements Searc public EntityViewEntity(EntityView entityView) { if (entityView.getId() != null) { - this.setId(entityView.getId().getId()); + this.setUuid(entityView.getId().getId()); } if (entityView.getEntityId() != null) { this.entityId = toString(entityView.getEntityId().getId()); @@ -136,8 +136,8 @@ public class EntityViewEntity extends BaseSqlEntity implements Searc @Override public EntityView toData() { - EntityView entityView = new EntityView(new EntityViewId(getId())); - entityView.setCreatedTime(UUIDs.unixTimestamp(getId())); + EntityView entityView = new EntityView(new EntityViewId(this.getUuid())); + entityView.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (entityId != null) { entityView.setEntityId(EntityIdFactory.getByTypeAndId(entityType.name(), toUUID(entityId).toString())); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index d775ce771c..6cade0f576 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -75,7 +75,7 @@ public class EventEntity extends BaseSqlEntity implements BaseEntity implements BaseEntity implements SearchT public RuleChainEntity(RuleChain ruleChain) { if (ruleChain.getId() != null) { - this.setId(ruleChain.getUuidId()); + this.setUuid(ruleChain.getUuidId()); } this.tenantId = toString(DaoUtil.getId(ruleChain.getTenantId())); this.name = ruleChain.getName(); @@ -100,8 +100,8 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT @Override public RuleChain toData() { - RuleChain ruleChain = new RuleChain(new RuleChainId(getId())); - ruleChain.setCreatedTime(UUIDs.unixTimestamp(getId())); + RuleChain ruleChain = new RuleChain(new RuleChainId(this.getUuid())); + ruleChain.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); ruleChain.setTenantId(new TenantId(toUUID(tenantId))); ruleChain.setName(name); if (firstRuleNodeId != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java index f36edc260b..1a7866418a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java @@ -69,7 +69,7 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex public RuleNodeEntity(RuleNode ruleNode) { if (ruleNode.getId() != null) { - this.setId(ruleNode.getUuidId()); + this.setUuid(ruleNode.getUuidId()); } if (ruleNode.getRuleChainId() != null) { this.ruleChainId = toString(DaoUtil.getId(ruleNode.getRuleChainId())); @@ -94,8 +94,8 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex @Override public RuleNode toData() { - RuleNode ruleNode = new RuleNode(new RuleNodeId(getId())); - ruleNode.setCreatedTime(UUIDs.unixTimestamp(getId())); + RuleNode ruleNode = new RuleNode(new RuleNodeId(this.getUuid())); + ruleNode.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (ruleChainId != null) { ruleNode.setRuleChainId(new RuleChainId(toUUID(ruleChainId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java index 239d3698b4..1da71224f0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java @@ -72,6 +72,12 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Column(name = ModelConstants.EMAIL_PROPERTY) private String email; + @Column(name = ModelConstants.TENANT_ISOLATED_TB_CORE) + private boolean isolatedTbCore; + + @Column(name = ModelConstants.TENANT_ISOLATED_TB_RULE_ENGINE) + private boolean isolatedTbRuleEngine; + @Type(type = "json") @Column(name = ModelConstants.TENANT_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; @@ -82,7 +88,7 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT public TenantEntity(Tenant tenant) { if (tenant.getId() != null) { - this.setId(tenant.getId().getId()); + this.setUuid(tenant.getId().getId()); } this.title = tenant.getTitle(); this.region = tenant.getRegion(); @@ -95,6 +101,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT this.phone = tenant.getPhone(); this.email = tenant.getEmail(); this.additionalInfo = tenant.getAdditionalInfo(); + this.isolatedTbCore = tenant.isIsolatedTbCore(); + this.isolatedTbRuleEngine = tenant.isIsolatedTbRuleEngine(); } @Override @@ -113,8 +121,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Override public Tenant toData() { - Tenant tenant = new Tenant(new TenantId(getId())); - tenant.setCreatedTime(UUIDs.unixTimestamp(getId())); + Tenant tenant = new Tenant(new TenantId(this.getUuid())); + tenant.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); tenant.setTitle(title); tenant.setRegion(region); tenant.setCountry(country); @@ -126,6 +134,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT tenant.setPhone(phone); tenant.setEmail(email); tenant.setAdditionalInfo(additionalInfo); + tenant.setIsolatedTbCore(isolatedTbCore); + tenant.setIsolatedTbRuleEngine(isolatedTbRuleEngine); return tenant; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java index 70593ca610..6957f87961 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java @@ -56,7 +56,7 @@ public final class UserCredentialsEntity extends BaseSqlEntity public UserCredentialsEntity(UserCredentials userCredentials) { if (userCredentials.getId() != null) { - this.setId(userCredentials.getId().getId()); + this.setUuid(userCredentials.getId().getId()); } if (userCredentials.getUserId() != null) { this.userId = toString(userCredentials.getUserId().getId()); @@ -69,8 +69,8 @@ public final class UserCredentialsEntity extends BaseSqlEntity @Override public UserCredentials toData() { - UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(getId())); - userCredentials.setCreatedTime(UUIDs.unixTimestamp(getId())); + UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(this.getUuid())); + userCredentials.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (userId != null) { userCredentials.setUserId(new UserId(toUUID(userId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java index 0100110daa..3af671583e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java @@ -81,7 +81,7 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< public UserEntity(User user) { if (user.getId() != null) { - this.setId(user.getId().getId()); + this.setUuid(user.getId().getId()); } this.authority = user.getAuthority(); if (user.getTenantId() != null) { @@ -108,8 +108,8 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< @Override public User toData() { - User user = new User(new UserId(getId())); - user.setCreatedTime(UUIDs.unixTimestamp(getId())); + User user = new User(new UserId(this.getUuid())); + user.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); user.setAuthority(authority); if (tenantId != null) { user.setTenantId(new TenantId(fromString(tenantId))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java index 8e81b30e8f..0cb69b9d8b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java @@ -62,7 +62,7 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement public WidgetTypeEntity(WidgetType widgetType) { if (widgetType.getId() != null) { - this.setId(widgetType.getId().getId()); + this.setUuid(widgetType.getId().getId()); } if (widgetType.getTenantId() != null) { this.tenantId = toString(widgetType.getTenantId().getId()); @@ -75,8 +75,8 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement @Override public WidgetType toData() { - WidgetType widgetType = new WidgetType(new WidgetTypeId(getId())); - widgetType.setCreatedTime(UUIDs.unixTimestamp(getId())); + WidgetType widgetType = new WidgetType(new WidgetTypeId(this.getUuid())); + widgetType.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { widgetType.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java index 7df6ea9fe7..75ea383090 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java @@ -55,7 +55,7 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl public WidgetsBundleEntity(WidgetsBundle widgetsBundle) { if (widgetsBundle.getId() != null) { - this.setId(widgetsBundle.getId().getId()); + this.setUuid(widgetsBundle.getId().getId()); } if (widgetsBundle.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(widgetsBundle.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java index dd56136e76..2da9915bb2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java @@ -132,10 +132,10 @@ public abstract class CassandraAbstractModelDao, D> exte protected EntityResultSet saveWithResult(TenantId tenantId, E entity) { log.debug("Save entity {}", entity); - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } else if (isDeleteOnSave()) { - removeById(tenantId, entity.getId()); + removeById(tenantId, entity.getUuid()); } Statement saveStatement = getSaveQuery(entity); saveStatement.setConsistencyLevel(cluster.getDefaultWriteConsistencyLevel()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index d3ce07f901..edc9c97a9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -58,8 +58,8 @@ public abstract class JpaAbstractDao, D> } setSearchText(entity); log.debug("Saving entity {}", entity); - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } entity = getCrudRepository().save(entity); return DaoUtil.getData(entity); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index aaddc15eb8..c4482a3247 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -69,7 +69,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im fromTimeUUID(tenantId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -86,7 +86,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im fromTimeUUID(customerId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -109,7 +109,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -121,7 +121,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java index c0a260e9da..566d9c3225 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java @@ -16,8 +16,6 @@ package org.thingsboard.server.dao.sql.audit; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; @@ -39,14 +37,11 @@ import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTimeDao; import org.thingsboard.server.dao.util.SqlDao; -import javax.annotation.PreDestroy; import javax.persistence.criteria.Predicate; import java.util.ArrayList; import java.util.List; import java.util.UUID; -import java.util.concurrent.Executors; -import static org.springframework.data.jpa.domain.Specifications.where; import static org.thingsboard.server.dao.model.ModelConstants.ID_PROPERTY; @Component @@ -118,8 +113,8 @@ public class JpaAuditLogDao extends JpaAbstractDao imp Specification timeSearchSpec = JpaAbstractSearchTimeDao.getTimeSearchPageSpec(pageLink, "id"); Specification fieldsSpec = getEntityFieldsSpec(tenantId, entityId, customerId, userId, actionTypes); Sort.Direction sortDirection = pageLink.isAscOrder() ? Sort.Direction.ASC : Sort.Direction.DESC; - Pageable pageable = new PageRequest(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); - return DaoUtil.convertDataList(auditLogRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); + Pageable pageable = PageRequest.of(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); + return DaoUtil.convertDataList(auditLogRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); } private Specification getEntityFieldsSpec(UUID tenantId, EntityId entityId, CustomerId customerId, UserId userId, List actionTypes) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java index c634e5a064..d8272e3a57 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java @@ -48,17 +48,17 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com } catch (Throwable throwable) { transactionManager.rollback(insertTransaction); if (throwable.getCause() instanceof ConstraintViolationException) { - log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getId(), entity.getName(), entity.getType()); + log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getUuid(), entity.getName(), entity.getType()); TransactionStatus transaction = getTransactionStatus(TransactionDefinition.PROPAGATION_REQUIRES_NEW); try { componentDescriptorEntity = processSaveOrUpdate(entity, insertOrUpdateOnUniqueKeyConflict); } catch (Throwable th) { - log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); + log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); transactionManager.rollback(transaction); } transactionManager.commit(transaction); } else { - log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); + log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); } } return componentDescriptorEntity; @@ -69,7 +69,7 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com protected Query getQuery(ComponentDescriptorEntity entity, String query) { return entityManager.createNativeQuery(query, ComponentDescriptorEntity.class) - .setParameter("id", UUIDConverter.fromTimeUUID(entity.getId())) + .setParameter("id", UUIDConverter.fromTimeUUID(entity.getUuid())) .setParameter("actions", entity.getActions()) .setParameter("clazz", entity.getClazz()) .setParameter("configuration_descriptor", entity.getConfigurationDescriptor().toString()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java index e281bdeec4..c33dd4f5e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java @@ -40,7 +40,7 @@ public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDe @Override protected ComponentDescriptorEntity doProcessSaveOrUpdate(ComponentDescriptorEntity entity, String query) { getQuery(entity, query).executeUpdate(); - return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getId())); + return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getUuid())); } private static String getInsertString(String conflictStatement) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java index 938b030d53..e66937166f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java @@ -94,7 +94,7 @@ public class JpaBaseComponentDescriptorDao extends JpaAbstractSearchTextDao deviceRepository.findByTenantId( fromTimeUUID(tenantId), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } else { return DaoUtil.convertDataList( deviceRepository.findByTenantId( fromTimeUUID(tenantId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } } @@ -95,7 +95,7 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao fromTimeUUID(customerId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -118,7 +118,7 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -130,7 +130,7 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 91a9181908..385a901dc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -70,7 +70,7 @@ public class JpaEntityViewDao extends JpaAbstractSearchTextDao timeSearchSpec = JpaAbstractSearchTimeDao.getTimeSearchPageSpec(pageLink, "id"); Specification fieldsSpec = getEntityFieldsSpec(tenantId, entityId, eventType); Sort.Direction sortDirection = pageLink.isAscOrder() ? Sort.Direction.ASC : Sort.Direction.DESC; - Pageable pageable = new PageRequest(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); - return DaoUtil.convertDataList(eventRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); + Pageable pageable = PageRequest.of(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); + return DaoUtil.convertDataList(eventRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); } @Override @@ -130,7 +129,7 @@ public class JpaBaseEventDao extends JpaAbstractSearchTimeDao timeSearchSpec = JpaAbstractSearchTimeDao.getTimeSearchPageSpec(pageLink, "toId"); Specification fieldsSpec = getEntityFieldsSpec(from, relationType, typeGroup, childType); Sort.Direction sortDirection = pageLink.isAscOrder() ? Sort.Direction.ASC : Sort.Direction.DESC; - Pageable pageable = new PageRequest(0, pageLink.getLimit(), sortDirection, "toId"); + Pageable pageable = PageRequest.of(0, pageLink.getLimit(), sortDirection, "toId"); return service.submit(() -> - DaoUtil.convertDataList(relationRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable).getContent())); + DaoUtil.convertDataList(relationRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable).getContent())); } private Specification getEntityFieldsSpec(EntityId from, String relationType, RelationTypeGroup typeGroup, EntityType childType) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 415cbd15c1..b1e4745223 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -60,7 +60,7 @@ public class JpaRuleChainDao extends JpaAbstractSearchTextDao region, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : UUIDConverter.fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 33b7c3c1e9..f30f6ba6d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -71,7 +71,7 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), Objects.toString(pageLink.getTextSearch(), ""), Authority.TENANT_ADMIN, - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -84,7 +84,7 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), Objects.toString(pageLink.getTextSearch(), ""), Authority.CUSTOMER_USER, - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 228c596642..af6785e69e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -68,7 +68,7 @@ public class JpaWidgetsBundleDao extends JpaAbstractSearchTextDao tsKvEntity.setStrKey(query.getKey())); return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index bf4cf5d9e6..0a2e210648 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -109,8 +109,8 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements keyId, query.getStartTs(), query.getEndTs(), - new PageRequest(0, query.getLimit(), - new Sort(Sort.Direction.fromString( + PageRequest.of(0, query.getLimit(), + Sort.by(Sort.Direction.fromString( query.getOrderBy()), "ts"))); timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey)); return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 9453fdb4c5..eb916cdb83 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; @@ -126,7 +127,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override public void deleteTenants() { log.trace("Executing deleteTenants"); - tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID),DEFAULT_TENANT_REGION); + tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID), DEFAULT_TENANT_REGION); } private DataValidator tenantValidator = @@ -140,19 +141,31 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe validateEmail(tenant.getEmail()); } } - }; + + @Override + protected void validateUpdate(TenantId tenantId, Tenant tenant) { + Tenant old = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing tenant!"); + } else if (old.isIsolatedTbRuleEngine() != tenant.isIsolatedTbRuleEngine()) { + throw new DataValidationException("Can't update isolatedTbRuleEngine property!"); + } else if (old.isIsolatedTbCore() != tenant.isIsolatedTbCore()) { + throw new DataValidationException("Can't update isolatedTbCore property!"); + } + } + }; private PaginatedRemover tenantsRemover = new PaginatedRemover() { - @Override - protected List findEntities(TenantId tenantId, String region, TextPageLink pageLink) { - return tenantDao.findTenantsByRegion(tenantId, region, pageLink); - } + @Override + protected List findEntities(TenantId tenantId, String region, TextPageLink pageLink) { + return tenantDao.findTenantsByRegion(tenantId, region, pageLink); + } - @Override - protected void removeEntity(TenantId tenantId, Tenant entity) { - deleteTenant(new TenantId(entity.getUuidId())); - } - }; + @Override + protected void removeEntity(TenantId tenantId, Tenant entity) { + deleteTenant(new TenantId(entity.getUuidId())); + } + }; } diff --git a/dao/src/main/resources/cassandra/schema-entities.cql b/dao/src/main/resources/cassandra/schema-entities.cql index de2b088cef..6d23ca8122 100644 --- a/dao/src/main/resources/cassandra/schema-entities.cql +++ b/dao/src/main/resources/cassandra/schema-entities.cql @@ -110,6 +110,8 @@ CREATE TABLE IF NOT EXISTS thingsboard.tenant ( phone text, email text, additional_info text, + isolated_tb_core boolean, + isolated_tb_rule_engine boolean, PRIMARY KEY (id, region) ); diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index 758aaafb10..f28f7f5ebd 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -183,7 +183,9 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255) + zip varchar(255), + isolated_tb_core boolean, + isolated_tb_rule_engine boolean ); CREATE TABLE IF NOT EXISTS user_credentials ( diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 55893fc124..36dac1b40e 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -183,7 +183,9 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255) + zip varchar(255), + isolated_tb_core boolean, + isolated_tb_rule_engine boolean ); CREATE TABLE IF NOT EXISTS user_credentials ( diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java index f02ae6b577..e0244c256e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java @@ -19,7 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/dao/src/test/resources/cassandra-test.properties b/dao/src/test/resources/cassandra-test.properties index af9a4b356f..51f34a08d6 100644 --- a/dao/src/test/resources/cassandra-test.properties +++ b/dao/src/test/resources/cassandra-test.properties @@ -60,3 +60,4 @@ cassandra.query.tenant_rate_limits.enabled=false cassandra.query.tenant_rate_limits.configuration=5000:1,100000:60 cassandra.query.tenant_rate_limits.print_tenant_names=false +service.type=monolith \ No newline at end of file diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 765e0da3d6..7b5f82da2a 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -16,6 +16,8 @@ spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver spring.datasource.hikari.maximumPoolSize = 50 +service.type=monolith + #database.ts.type=timescale #database.ts.type=sql #database.entities.type=sql @@ -33,4 +35,12 @@ spring.datasource.hikari.maximumPoolSize = 50 #spring.datasource.password=postgres #spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest #spring.datasource.driverClassName=org.postgresql.Driver -#spring.datasource.hikari.maximumPoolSize = 50 \ No newline at end of file +#spring.datasource.hikari.maximumPoolSize = 50 + +queue.rule-engine.queues[0].name=Main +queue.rule-engine.queues[0].topic=tb_rule_engine.main +queue.rule-engine.queues[0].poll-interval=25 +queue.rule-engine.queues[0].partitions=3 +queue.rule-engine.queues[0].pack-processing-timeout=3000 +queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES +queue.rule-engine.queues[0].submit-strategy.type=BURST \ No newline at end of file diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index c17b1ddde4..f0facf8cc1 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -19,6 +19,7 @@ const COMPILATION_ERROR = 0; const RUNTIME_ERROR = 1; const TIMEOUT_ERROR = 2; const UNRECOGNIZED = -1; +let headers; const config = require('config'), logger = require('../config/logger')._logger('JsInvokeMessageProcessor'), @@ -43,6 +44,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { var responseTopic; try { var request = JSON.parse(message.value.toString('utf8')); + headers = message.headers; var buf = message.headers['requestId']; requestId = Utils.UUIDFromBuffer(buf); buf = message.headers['responseTopic']; @@ -148,7 +150,8 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT messages: [ { key: scriptId, - value: rawResponse + value: rawResponse, + headers: headers } ] } diff --git a/pom.xml b/pom.xml index 6725766840..8876e68538 100755 --- a/pom.xml +++ b/pom.xml @@ -30,11 +30,11 @@ ${basedir} thingsboard - 2.1.3.RELEASE - 5.1.5.RELEASE - 5.1.4.RELEASE - 2.1.5.RELEASE - 2.9.0 + 2.2.4.RELEASE + 5.2.2.RELEASE + 5.2.2.RELEASE + 2.2.4.RELEASE + 3.1.0 0.7.0 2.2.0 4.12 @@ -51,12 +51,12 @@ 1.6 2.5 1.4 - 2.9.9.3 - 2.9.9 - 2.9.9 + 2.10.2 + 2.10.2 + 2.10.2 2.2.6 - 2.11 - 2.4.2 + 2.13 + 2.6.3 1.0.2 2.6.2 1.7 @@ -68,7 +68,7 @@ 1.22.1 1.16.18 1.1.0 - 4.1.37.Final + 4.1.45.Final 1.5.0 4.8.0 2.19.1 @@ -77,7 +77,7 @@ 1.0.0 0.7 1.15.0 - 1.56 + 1.64 2.0.1 2.5.0 2.5.3 @@ -92,9 +92,15 @@ 4.1.1 2.57 2.7.7 - 1.23 + 1.25 + 1.3.10 + 1.11.747 + 1.84.0 + 3.2.0 1.5.0 1.4.3 + 1.9.4 + 3.2.2 @@ -887,6 +893,21 @@ jts-core ${jts.version} + + com.amazonaws + aws-java-sdk-sqs + ${amazonaws.sqs.version} + + + com.google.cloud + google-cloud-pubsub + ${pubsub.client.version} + + + com.microsoft.azure + azure-servicebus + ${azure-servicebus.version} + org.passay passay @@ -897,6 +918,37 @@ uap-java ${ua-parser.version} + + commons-beanutils + commons-beanutils + ${commons-beanutils.version} + + + commons-collections + commons-collections + ${commons-collections.version} + + + org.yaml + snakeyaml + ${snakeyaml.version} + + + org.apache.struts + struts-core + ${struts.version} + + + org.apache.struts + struts-taglib + ${struts.version} + + + org.apache.struts + struts-tiles + ${struts.version} + + diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 652de133b7..e14996d12f 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -45,7 +45,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -98,7 +98,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java deleted file mode 100644 index bb0210b1ad..0000000000 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.api; - -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -import java.util.function.Consumer; - -public interface RuleChainTransactionService { - - void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure); - - void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure); - - void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] bytes); - -} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java index 34bfe4d3f4..7cab689d13 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java @@ -18,6 +18,7 @@ package org.thingsboard.rule.engine.api; import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -28,11 +29,11 @@ import java.util.UUID; @Builder public final class RuleEngineDeviceRpcRequest { + private final TenantId tenantId; private final DeviceId deviceId; private final int requestId; private final UUID requestUUID; - private final String originHost; - private final int originPort; + private final String originServiceId; private final boolean oneway; private final String method; private final String body; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java index 69ce133c5d..baac594a17 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java @@ -16,6 +16,8 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.data.id.DeviceId; + +import java.util.UUID; import java.util.function.Consumer; /** @@ -23,8 +25,8 @@ import java.util.function.Consumer; */ public interface RuleEngineRpcService { - void sendRpcReply(DeviceId deviceId, int requestId, String body); + void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body); - void sendRpcRequest(RuleEngineDeviceRpcRequest request, Consumer consumer); + void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest request, Consumer consumer); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java index f57849fa35..d2e19652a2 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java @@ -44,6 +44,4 @@ public interface RuleEngineTelemetryService { void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, boolean value, FutureCallback callback); - void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes); - } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 38914f4553..9b1edc5f5c 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.alarm.AlarmService; @@ -45,27 +44,91 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import java.util.Set; +import java.util.function.Consumer; /** * Created by ashvayka on 13.01.18. */ public interface TbContext { + /* + * + * METHODS TO CONTROL THE MESSAGE FLOW + * + */ + + /** + * Indicates that message was successfully processed by the rule node. + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using "Success" relationType. + * + * @param msg + */ + void tellSuccess(TbMsg msg); + + /** + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using specified relationType. + * + * @param msg + * @param relationType + */ void tellNext(TbMsg msg, String relationType); - void tellNext(TbMsg msg, String relationType, Throwable th); - + /** + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using one of specified relationTypes. + * + * @param msg + * @param relationTypes + */ void tellNext(TbMsg msg, Set relationTypes); + /** + * Sends message to the current Rule Node with specified delay in milliseconds. + * Note: this message is not queued and may be lost in case of a server restart. + * + * @param msg + */ void tellSelf(TbMsg msg, long delayMs); - boolean isLocalEntity(EntityId entityId); - + /** + * Notifies Rule Engine about failure to process current message. + * + * @param msg - message + * @param th - exception + */ void tellFailure(TbMsg msg, Throwable th); - void updateSelf(RuleNode self); + /** + * Puts new message to queue for processing by the Root Rule Chain + * + * @param msg - message + */ + void enqueue(TbMsg msg, Runnable onSuccess, Consumer onFailure); + + /** + * Puts new message to custom queue for processing + * + * @param msg - message + */ + void enqueue(TbMsg msg, String queueName, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellFailure(TbMsg msg, String failureMessage); + + void enqueueForTellNext(TbMsg msg, String relationType); + + void enqueueForTellNext(TbMsg msg, Set relationTypes); + + void enqueueForTellNext(TbMsg msg, String relationType, Runnable onSuccess, Consumer onFailure); - void sendTbMsgToRuleEngine(TbMsg msg); + void enqueueForTellNext(TbMsg msg, Set relationTypes, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellNext(TbMsg msg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellNext(TbMsg msg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure); + + void ack(TbMsg tbMsg); TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data); @@ -77,8 +140,17 @@ public interface TbContext { TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId); + // TODO: Does this changes the message? TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId); + /* + * + * METHODS TO PROCESS THE MESSAGES + * + */ + + boolean isLocalEntity(EntityId entityId); + RuleNodeId getSelfId(); TenantId getTenantId(); @@ -129,9 +201,7 @@ public interface TbContext { void logJsEvalFailure(); - String getNodeId(); - - RuleChainTransactionService getRuleChainTransactionService(); + String getServiceId(); EventLoopGroup getSharedEventLoop(); @@ -139,7 +209,7 @@ public interface TbContext { ResultSetFuture submitCassandraTask(CassandraStatementTask task); + @Deprecated RedisTemplate getRedisTemplate(); - String getServerAddress(); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java index 26f7d14f8b..85fc75139b 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java @@ -16,7 +16,7 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import java.util.concurrent.ExecutionException; @@ -31,6 +31,6 @@ public interface TbNode { void destroy(); - default void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) {} + default void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) {} } diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 373e4107f8..645419ca41 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -35,8 +35,7 @@ UTF-8 ${basedir}/../.. - 1.11.323 - 1.83.0 + 1.11.747 1.16.0 @@ -99,7 +98,6 @@ com.google.cloud google-cloud-pubsub - ${pubsub.client.version} com.google.api.grpc diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java index 6320d5625f..b3344105bb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java @@ -29,8 +29,6 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import javax.script.ScriptException; - import static org.thingsboard.common.util.DonAsynchron.withCallback; @@ -63,16 +61,18 @@ public abstract class TbAbstractAlarmNode ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Created"), + throwable -> ctx.tellFailure(toAlarmMsg(ctx, alarmResult, msg), throwable)); } else if (alarmResult.isUpdated) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Updated"); } else if (alarmResult.isCleared) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Cleared"); + } else { + ctx.tellSuccess(msg); } }, - t -> ctx.tellFailure(msg, t) - , ctx.getDbCallbackExecutor()); + t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } protected abstract ListenableFuture processAlarm(TbContext ctx, TbMsg msg); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java index 3ee2127fde..bdf8c5fe1e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java @@ -63,7 +63,7 @@ public abstract class TbAbstractCustomerActionNode ctx.tellNext(msg, "Success"), + m -> ctx.tellSuccess(msg), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } @@ -122,7 +122,9 @@ public abstract class TbAbstractCustomerActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), + throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); return Optional.of(savedCustomer.getId()); } return Optional.empty(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java index 3d2cb5d8c8..5ec91bdf0e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java @@ -187,7 +187,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Device Created message: {}", savedDevice), + throwable -> log.warn("Failed to push Device Created message: {}", savedDevice, throwable)); targetEntity.setEntityId(savedDevice.getId()); } break; @@ -202,7 +204,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Asset Created message: {}", savedAsset), + throwable -> log.warn("Failed to push Asset Created message: {}", savedAsset, throwable)); targetEntity.setEntityId(savedAsset.getId()); } break; @@ -216,7 +220,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), + throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); targetEntity.setEntityId(savedCustomer.getId()); } break; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java index f009b737e9..c5dc5cc5f4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java @@ -89,7 +89,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { if ((endTime != 0 && endTime > now && startTime < now) || (endTime == 0 && startTime < now)) { if (DataConstants.ATTRIBUTES_UPDATED.equals(msg.getType()) || DataConstants.ACTIVITY_EVENT.equals(msg.getType()) || - SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType()) ) { + SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType())) { Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(msg.getData())); List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr.getKey(), entityView)).collect(Collectors.toList()); @@ -117,13 +117,14 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr, entityView)).collect(Collectors.toList()); - if (filteredAttributes != null && !filteredAttributes.isEmpty()) { + if (!filteredAttributes.isEmpty()) { ctx.getAttributesService().removeAll(ctx.getTenantId(), entityView.getId(), scope, filteredAttributes); transformAndTellNext(ctx, msg, entityView); } } } } + ctx.ack(msg); }, t -> ctx.tellFailure(msg, t)); } else { @@ -135,8 +136,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } private void transformAndTellNext(TbContext ctx, TbMsg msg, EntityView entityView) { - TbMsg updMsg = ctx.transformMsg(msg, msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()); - ctx.tellNext(updMsg, SUCCESS); + ctx.enqueueForTellNext(ctx.newMsg(msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()), SUCCESS); } private boolean attributeContainsInEntityView(String scope, String attrKey, EntityView entityView) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java index 62410e3052..2831e890d6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java @@ -58,7 +58,7 @@ public class TbLogNode implements TbNode { toString -> { ctx.logJsEvalResponse(); log.info(toString); - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); }, t -> { ctx.logJsEvalResponse(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 2e99e75b53..19d8515455 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -68,18 +68,19 @@ public class TbMsgCountNode implements TbNode { public void onMsg(TbContext ctx, TbMsg msg) { if (msg.getType().equals(TB_MSG_COUNT_NODE_MSG) && msg.getId().equals(nextTickId)) { JsonObject telemetryJson = new JsonObject(); - telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getNodeId(), messagesProcessed.longValue()); + telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getServiceId(), messagesProcessed.longValue()); messagesProcessed = new AtomicLong(0); TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("delta", Long.toString(System.currentTimeMillis() - lastScheduledTs + delay)); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, TbMsgDataType.JSON, gson.toJson(telemetryJson), null, null, 0L); - ctx.tellNext(tbMsg, SUCCESS); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, gson.toJson(telemetryJson)); + ctx.enqueueForTellNext(tbMsg, SUCCESS); scheduleTickMsg(ctx); } else { messagesProcessed.incrementAndGet(); + ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 50471b366a..aa7bfa81d1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -105,10 +105,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { - withCallback(save(msg, ctx), aVoid -> { - ctx.tellNext(msg, SUCCESS); - }, e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); + public void onMsg(TbContext ctx, TbMsg msg) { + withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java index 59d27018d7..d43ebb26ef 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java @@ -74,11 +74,8 @@ public class TbSnsNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), - t -> { - TbMsg next = processException(ctx, msg, t); - ctx.tellFailure(next, t); - }); + ctx::tellSuccess, + t -> ctx.tellFailure(processException(ctx, msg, t), t)); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java index 37f4e4baee..3cc6074165 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java @@ -80,13 +80,10 @@ public class TbSqsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), - t -> { - TbMsg next = processException(ctx, msg, t); - ctx.tellFailure(next, t); - }); + ctx::tellSuccess, + t -> ctx.tellFailure(processException(ctx, msg, t), t)); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index cd566f8dd2..4f469678a2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -74,7 +74,7 @@ public class TbMsgGeneratorNode implements TbNode { } @Override - public void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) { + public void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) { updateGeneratorState(ctx); } @@ -97,7 +97,7 @@ public class TbMsgGeneratorNode implements TbNode { withCallback(generate(ctx), m -> { if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { - ctx.tellNext(m, SUCCESS); + ctx.enqueueForTellNext(m, SUCCESS); scheduleTickMsg(ctx); currentMsgCount++; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java index 0ef9c4d46a..1807c5c4fe 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java @@ -65,15 +65,16 @@ public class TbMsgDelayNode implements TbNode { if (msg.getType().equals(TB_MSG_DELAY_NODE_MSG)) { TbMsg pendingMsg = pendingMsgs.remove(UUID.fromString(msg.getData())); if (pendingMsg != null) { - ctx.tellNext(pendingMsg, SUCCESS); + ctx.enqueueForTellNext(pendingMsg, SUCCESS); } } else { - if(pendingMsgs.size() < config.getMaxPendingMsgs()) { + if (pendingMsgs.size() < config.getMaxPendingMsgs()) { pendingMsgs.put(msg.getId(), msg); TbMsg tickMsg = ctx.newMsg(TB_MSG_DELAY_NODE_MSG, ctx.getSelfId(), new TbMsgMetaData(), msg.getId().toString()); ctx.tellSelf(tickMsg, getDelay(msg)); + ctx.ack(msg); } else { - ctx.tellNext(msg, FAILURE, new RuntimeException("Max limit of pending messages reached!")); + ctx.tellFailure(msg, new RuntimeException("Max limit of pending messages reached!")); } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java index 086e3aded0..dc0eb9f396 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java @@ -72,14 +72,13 @@ public class TbCheckAlarmStatusNode implements TbNode { break; } } - if (isPresent) { ctx.tellNext(msg, "True"); } else { ctx.tellNext(msg, "False"); } } else { - ctx.tellFailure(msg, new TbNodeException("No such Alarm found.")); + ctx.tellFailure(msg, new TbNodeException("No such alarm found.")); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java index dd55ef2584..3799e60f73 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java @@ -44,7 +44,7 @@ public class TbMsgTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { ctx.tellNext(msg, config.getMessageTypes().contains(msg.getType()) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java index d0fe593711..07a0508a87 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java @@ -45,7 +45,7 @@ public class TbMsgTypeSwitchNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String relationType; if (msg.getType().equals(SessionMsgType.POST_ATTRIBUTES_REQUEST.name())) { relationType = "Post attributes"; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java index ac3db9968f..de3a48142e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java @@ -42,7 +42,7 @@ public class TbOriginatorTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { EntityType originatorType = msg.getOriginator().getEntityType(); ctx.tellNext(msg, config.getOriginatorTypes().contains(originatorType) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java new file mode 100644 index 0000000000..b5af3f1563 --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.flow; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.EmptyNodeConfiguration; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.msg.TbMsg; + +@Slf4j +@RuleNode( + type = ComponentType.ACTION, + name = "acknowledge", + configClazz = EmptyNodeConfiguration.class, + nodeDescription = "Acknowledges the incoming message", + nodeDetails = "After acknowledgement, the message is pushed to related rule nodes. Useful if you don't care what happens to this message next.", + uiResources = {"static/rulenode/rulenode-core-config.js"}, + configDirective = "tbNodeEmptyConfig" +) +public class TbAckNode implements TbNode { + + EmptyNodeConfiguration config; + + @Override + public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); + } + + @Override + public void onMsg(TbContext ctx, TbMsg msg) { + ctx.ack(msg); + ctx.tellSuccess(msg); + } + + @Override + public void destroy() { + } +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java new file mode 100644 index 0000000000..d5048eadbd --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.flow; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.TbRelationTypes; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.msg.TbMsg; + +import static org.thingsboard.common.util.DonAsynchron.withCallback; + +@Slf4j +@RuleNode( + type = ComponentType.ACTION, + name = "checkpoint", + configClazz = TbCheckpointNodeConfiguration.class, + nodeDescription = "transfers the message to another queue", + nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.", + uiResources = {"static/rulenode/rulenode-core-config.js"}, + configDirective = "tbActionNodeCheckPointConfig" +) +public class TbCheckpointNode implements TbNode { + + private TbCheckpointNodeConfiguration config; + + @Override + public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, TbCheckpointNodeConfiguration.class); + } + + @Override + public void onMsg(TbContext ctx, TbMsg msg) { + ctx.enqueueForTellNext(msg, config.getQueueName(), TbRelationTypes.SUCCESS, () -> ctx.ack(msg), error -> ctx.tellFailure(msg, error)); + } + + @Override + public void destroy() { + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java rename to rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java index f49b2bdae8..eb5b8e29d9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java @@ -13,25 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.actors.device; +package org.thingsboard.rule.engine.flow; -import akka.actor.ActorRef; import lombok.Data; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.rule.engine.api.NodeConfiguration; -/** - * Created by ashvayka on 15.03.18. - */ @Data -public final class DeviceActorToRuleEngineMsg implements TbActorMsg { +public class TbCheckpointNodeConfiguration implements NodeConfiguration { - private final ActorRef callbackRef; - private final TbMsg tbMsg; + private String queueName; @Override - public MsgType getMsgType() { - return MsgType.DEVICE_ACTOR_TO_RULE_ENGINE_MSG; + public TbCheckpointNodeConfiguration defaultConfiguration() { + TbCheckpointNodeConfiguration configuration = new TbCheckpointNodeConfiguration(); + configuration.setQueueName("HighPriority"); + return configuration; } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java index 8e3bcdb7dc..5a8b92cf80 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java @@ -99,7 +99,7 @@ public class TbPubSubNode implements TbNode { ApiFutures.addCallback(messageIdFuture, new ApiFutureCallback() { public void onSuccess(String messageId) { TbMsg next = processPublishResult(ctx, msg, messageId); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + ctx.tellSuccess(next); } public void onFailure(Throwable t) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java index fe765abdf9..832a86b5cb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java @@ -47,11 +47,12 @@ import java.util.concurrent.TimeoutException; type = ComponentType.ACTION, name = "gps geofencing events", configClazz = TbGpsGeofencingActionNodeConfiguration.class, - relationTypes = {"Entered", "Left", "Inside", "Outside"}, + relationTypes = {"Success", "Entered", "Left", "Inside", "Outside"}, nodeDescription = "Produces incoming messages using GPS based geofencing", nodeDetails = "Extracts latitude and longitude parameters from incoming message and returns different events based on configuration parameters", uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbActionNodeGpsGeofencingConfig") + configDirective = "tbActionNodeGpsGeofencingConfig" +) public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { private final Map entityStates = new HashMap<>(); @@ -66,7 +67,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { try { Optional entry = ctx.getAttributesService() - .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getNodeId()) + .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getServiceId()) .get(1, TimeUnit.MINUTES); if (entry.isPresent()) { JsonObject element = parser.parse(entry.get().getValueAsString()).getAsJsonObject(); @@ -78,17 +79,26 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode (entityState.isInside() ? - TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { - setStaid(ctx, msg.getOriginator(), entityState); - ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); + told = true; + } else { + if (!entityState.isStayed()) { + long stayTime = ts - entityState.getStateSwitchTime(); + if (stayTime > (entityState.isInside() ? + TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { + setStaid(ctx, msg.getOriginator(), entityState); + ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); + told = true; + } } } + if (!told) { + ctx.tellSuccess(msg); + } } private void switchState(TbContext ctx, EntityId entityId, EntityGeofencingState entityState, boolean matches, long ts) { @@ -108,7 +118,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode attributeKvEntryList = Collections.singletonList(entry); ctx.getAttributesService().save(ctx.getTenantId(), entityId, DataConstants.SERVER_SCOPE, attributeKvEntryList); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java index 267e8711aa..4955ebc400 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java @@ -17,12 +17,21 @@ package org.thingsboard.rule.engine.kafka; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.BooleanUtils; -import org.apache.kafka.clients.producer.*; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.header.internals.RecordHeaders; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.TbRelationTypes; import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -30,7 +39,6 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Properties; -import java.util.concurrent.ExecutionException; @Slf4j @RuleNode( @@ -63,7 +71,7 @@ public class TbKafkaNode implements TbNode { public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { this.config = TbNodeUtils.convert(configuration, TbKafkaNodeConfiguration.class); Properties properties = new Properties(); - properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getNodeId()); + properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getServiceId()); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getBootstrapServers()); properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, config.getValueSerializer()); properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, config.getKeySerializer()); @@ -73,8 +81,7 @@ public class TbKafkaNode implements TbNode { properties.put(ProducerConfig.LINGER_MS_CONFIG, config.getLinger()); properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, config.getBufferMemory()); if (config.getOtherProperties() != null) { - config.getOtherProperties() - .forEach(properties::put); + config.getOtherProperties().forEach(properties::put); } addMetadataKeyValuesAsKafkaHeaders = BooleanUtils.toBooleanDefaultIfNull(config.isAddMetadataKeyValuesAsKafkaHeaders(), false); toBytesCharset = config.getKafkaHeadersCharset() != null ? Charset.forName(config.getKafkaHeadersCharset()) : StandardCharsets.UTF_8; @@ -86,7 +93,7 @@ public class TbKafkaNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String topic = TbNodeUtils.processPattern(config.getTopicPattern(), msg.getMetaData()); try { if (!addMetadataKeyValuesAsKafkaHeaders) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java index 1c41c2a124..ee46b81116 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.util.Properties; import static org.thingsboard.common.util.DonAsynchron.withCallback; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @Slf4j @RuleNode( @@ -79,7 +78,7 @@ public class TbSendEmailNode implements TbNode { sendEmail(ctx, email); return null; }), - ok -> ctx.tellNext(msg, SUCCESS), + ok -> ctx.tellSuccess(msg), fail -> ctx.tellFailure(msg, fail)); } catch (Exception ex) { ctx.tellFailure(msg, ex); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 0bf1c23cc7..8aa15a4f38 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -43,7 +43,6 @@ import java.util.stream.Collectors; import static org.thingsboard.common.util.DonAsynchron.withCallback; import static org.thingsboard.rule.engine.api.TbRelationTypes.FAILURE; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; import static org.thingsboard.server.common.data.DataConstants.CLIENT_SCOPE; import static org.thingsboard.server.common.data.DataConstants.LATEST_TS; import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; @@ -101,7 +100,7 @@ public abstract class TbAbstractGetAttributesNode ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java index 2906d3922f..43738fd706 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java @@ -59,7 +59,7 @@ public abstract class TbAbstractGetEntityDetailsNode ctx.tellNext(m, SUCCESS), + ctx::tellSuccess, t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java index bc19d6c56f..c7387a5ed0 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java @@ -51,8 +51,7 @@ public abstract class TbEntityGetAttrNode implements TbNode @Override public void onMsg(TbContext ctx, TbMsg msg) { try { - withCallback( - findEntityAsync(ctx, msg.getOriginator()), + withCallback(findEntityAsync(ctx, msg.getOriginator()), entityId -> safeGetAttributes(ctx, msg, entityId), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { @@ -89,7 +88,7 @@ public abstract class TbEntityGetAttrNode implements TbNode String attrName = config.getAttrMapping().get(r.getKey()); msg.getMetaData().putValue(attrName, r.getValueAsString()); }); - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java index 717cb207c4..12577076d2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java @@ -54,10 +54,10 @@ public class TbGetOriginatorFieldsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { try { withCallback(putEntityFields(ctx, msg.getOriginator(), msg), - i -> ctx.tellNext(msg, SUCCESS), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); + i -> ctx.tellSuccess(msg), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { ctx.tellFailure(msg, th); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index caef71a4db..a3d84b25db 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -106,8 +106,7 @@ public class TbGetTelemetryNode implements TbNode { ListenableFuture> list = ctx.getTimeseriesService().findAll(ctx.getTenantId(), msg.getOriginator(), buildQueries(msg)); DonAsynchron.withCallback(list, data -> { process(data, msg); - TbMsg newMsg = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData()); - ctx.tellNext(newMsg, SUCCESS); + ctx.tellSuccess(ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData())); }, error -> ctx.tellFailure(msg, error), ctx.getDbCallbackExecutor()); } catch (Exception e) { ctx.tellFailure(msg, e); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java index 755cd27a33..b13f72b238 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java @@ -72,12 +72,12 @@ public class TbMqttNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String topic = TbNodeUtils.processPattern(this.config.getTopicPattern(), msg.getMetaData()); this.mqttClient.publish(topic, Unpooled.wrappedBuffer(msg.getData().getBytes(UTF8)), MqttQoS.AT_LEAST_ONCE) .addListener(future -> { if (future.isSuccess()) { - ctx.tellNext(msg, TbRelationTypes.SUCCESS); + ctx.tellSuccess(msg); } else { TbMsg next = processException(ctx, msg, future.cause()); ctx.tellFailure(next, future.cause()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java index 3c3bde2915..4a61cf63f3 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java @@ -74,9 +74,9 @@ public class TbRabbitMqNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), + ctx::tellSuccess, t -> { TbMsg next = processException(ctx, msg, t); ctx.tellFailure(next, t); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 65eea0b4c9..ba860c103b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.http.client.Netty4ClientHttpRequestFactory; import org.springframework.util.concurrent.ListenableFuture; @@ -84,7 +83,7 @@ class TbHttpClient { } } - void processMessage(TbContext ctx, TbMsg msg, TbRedisQueueProcessor queueProcessor) { + void processMessage(TbContext ctx, TbMsg msg) { String endpointUrl = TbNodeUtils.processPattern(config.getRestEndpointUrlPattern(), msg.getMetaData()); HttpHeaders headers = prepareHeaders(msg.getMetaData()); HttpMethod method = HttpMethod.valueOf(config.getRequestMethod()); @@ -95,13 +94,6 @@ class TbHttpClient { future.addCallback(new ListenableFutureCallback>() { @Override public void onFailure(Throwable throwable) { - if (config.isUseRedisQueueForMsgPersistence()) { - if (throwable instanceof HttpClientErrorException) { - processHttpClientError(((HttpClientErrorException) throwable).getStatusCode(), msg, queueProcessor); - } else { - queueProcessor.pushOnFailure(msg); - } - } TbMsg next = processException(ctx, msg, throwable); ctx.tellFailure(next, throwable); } @@ -109,15 +101,9 @@ class TbHttpClient { @Override public void onSuccess(ResponseEntity responseEntity) { if (responseEntity.getStatusCode().is2xxSuccessful()) { - if (config.isUseRedisQueueForMsgPersistence()) { - queueProcessor.resetCounter(); - } TbMsg next = processResponse(ctx, msg, responseEntity); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + ctx.tellSuccess(next); } else { - if (config.isUseRedisQueueForMsgPersistence()) { - processHttpClientError(responseEntity.getStatusCode(), msg, queueProcessor); - } TbMsg next = processFailureResponse(ctx, msg, responseEntity); ctx.tellNext(next, TbRelationTypes.FAILURE); } @@ -183,11 +169,4 @@ class TbHttpClient { } } - private void processHttpClientError(HttpStatus statusCode, TbMsg msg, TbRedisQueueProcessor queueProcessor) { - if (statusCode.is4xxClientError()) { - log.warn("[{}] Client error during message delivering!", msg); - } else { - queueProcessor.pushOnFailure(msg); - } - } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java deleted file mode 100644 index cff9faaab5..0000000000 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.rest; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.data.redis.core.ListOperations; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.server.common.msg.TbMsg; - -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -@Data -@Slf4j -class TbRedisQueueProcessor { - - private static final int MAX_QUEUE_SIZE = Integer.MAX_VALUE; - - private final TbContext ctx; - private final TbHttpClient httpClient; - private final ExecutorService executor; - private final ListOperations listOperations; - private final String redisKey; - private final boolean trimQueue; - private final int maxQueueSize; - - private AtomicInteger failuresCounter; - private Future future; - - TbRedisQueueProcessor(TbContext ctx, TbHttpClient httpClient, boolean trimQueue, int maxQueueSize) { - this.ctx = ctx; - this.httpClient = httpClient; - this.executor = Executors.newSingleThreadExecutor(); - this.listOperations = ctx.getRedisTemplate().opsForList(); - this.redisKey = constructRedisKey(); - this.trimQueue = trimQueue; - this.maxQueueSize = maxQueueSize; - init(); - } - - private void init() { - failuresCounter = new AtomicInteger(0); - future = executor.submit(() -> { - while (true) { - if (failuresCounter.get() != 0 && failuresCounter.get() % 50 == 0) { - sleep("Target HTTP server is down...", 3); - } - if (listOperations.size(redisKey) > 0) { - List list = listOperations.range(redisKey, -10, -1); - list.forEach(obj -> { - TbMsg msg = TbMsg.fromBytes((byte[]) obj); - log.debug("Trying to send the message: {}", msg); - listOperations.remove(redisKey, -1, obj); - httpClient.processMessage(ctx, msg, this); - }); - } else { - sleep("Queue is empty, waiting for tasks!", 1); - } - } - }); - } - - void destroy() { - if (future != null) { - future.cancel(true); - } - if (executor != null) { - executor.shutdownNow(); - } - } - - void push(TbMsg msg) { - listOperations.leftPush(redisKey, TbMsg.toByteArray(msg)); - if (trimQueue) { - listOperations.trim(redisKey, 0, validateMaxQueueSize()); - } - } - - void pushOnFailure(TbMsg msg) { - listOperations.rightPush(redisKey, TbMsg.toByteArray(msg)); - failuresCounter.incrementAndGet(); - } - - void resetCounter() { - failuresCounter.set(0); - } - - private String constructRedisKey() { - return ctx.getServerAddress() + ctx.getSelfId(); - } - - private int validateMaxQueueSize() { - if (maxQueueSize != 0) { - return maxQueueSize; - } - return MAX_QUEUE_SIZE; - } - - private void sleep(String logMessage, int sleepSeconds) { - try { - log.debug(logMessage); - TimeUnit.SECONDS.sleep(sleepSeconds); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted!", e); - } - } -} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java index 9b9d275977..9cb171d0dc 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java @@ -25,8 +25,6 @@ import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import java.util.concurrent.ExecutionException; - @Slf4j @RuleNode( type = ComponentType.EXTERNAL, @@ -47,7 +45,6 @@ public class TbRestApiCallNode implements TbNode { private boolean useRedisQueueForMsgPersistence; private TbHttpClient httpClient; - private TbRedisQueueProcessor queueProcessor; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -55,20 +52,13 @@ public class TbRestApiCallNode implements TbNode { httpClient = new TbHttpClient(config); useRedisQueueForMsgPersistence = config.isUseRedisQueueForMsgPersistence(); if (useRedisQueueForMsgPersistence) { - if (ctx.getRedisTemplate() == null) { - throw new RuntimeException("Redis cache type must be used!"); - } - queueProcessor = new TbRedisQueueProcessor(ctx, httpClient, config.isTrimQueue(), config.getMaxQueueSize()); + log.warn("[{}][{}] Usage of Redis Template is deprecated starting 2.5 and will have no affect", ctx.getTenantId(), ctx.getSelfId()); } } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { - if (useRedisQueueForMsgPersistence) { - queueProcessor.push(msg); - } else { - httpClient.processMessage(ctx, msg, null); - } + public void onMsg(TbContext ctx, TbMsg msg) { + httpClient.processMessage(ctx, msg); } @Override @@ -76,9 +66,6 @@ public class TbRestApiCallNode implements TbNode { if (this.httpClient != null) { this.httpClient.destroy(); } - if (this.queueProcessor != null) { - this.queueProcessor.destroy(); - } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 9444b52210..145e73d450 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -28,6 +28,8 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; +import java.util.UUID; + @Slf4j @RuleNode( type = ComponentType.ACTION, @@ -50,15 +52,22 @@ public class TbSendRPCReplyNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { + String serviceIdStr = msg.getMetaData().getValue(config.getServiceIdMetaDataAttribute()); + String sessionIdStr = msg.getMetaData().getValue(config.getSessionIdMetaDataAttribute()); String requestIdStr = msg.getMetaData().getValue(config.getRequestIdMetaDataAttribute()); if (msg.getOriginator().getEntityType() != EntityType.DEVICE) { ctx.tellFailure(msg, new RuntimeException("Message originator is not a device entity!")); } else if (StringUtils.isEmpty(requestIdStr)) { ctx.tellFailure(msg, new RuntimeException("Request id is not present in the metadata!")); + } else if (StringUtils.isEmpty(serviceIdStr)) { + ctx.tellFailure(msg, new RuntimeException("Service id is not present in the metadata!")); + } else if (StringUtils.isEmpty(sessionIdStr)) { + ctx.tellFailure(msg, new RuntimeException("Session id is not present in the metadata!")); } else if (StringUtils.isEmpty(msg.getData())) { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { - ctx.getRpcService().sendRpcReply(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); + ctx.getRpcService().sendRpcReplyToDevice(serviceIdStr, UUID.fromString(sessionIdStr), Integer.parseInt(requestIdStr), msg.getData()); + ctx.tellSuccess(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java index 9aa94905e6..ad1d399e76 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java @@ -16,8 +16,6 @@ package org.thingsboard.rule.engine.rpc; import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.JsonNode; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -38,7 +36,6 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import java.io.IOException; import java.util.Random; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -86,10 +83,8 @@ public class TbSendRPCRequestNode implements TbNode { tmp = msg.getMetaData().getValue("requestUUID"); UUID requestUUID = !StringUtils.isEmpty(tmp) ? UUID.fromString(tmp) : UUIDs.timeBased(); - tmp = msg.getMetaData().getValue("originHost"); - String originHost = !StringUtils.isEmpty(tmp) ? tmp : null; - tmp = msg.getMetaData().getValue("originPort"); - int originPort = !StringUtils.isEmpty(tmp) ? Integer.parseInt(tmp) : 0; + tmp = msg.getMetaData().getValue("originServiceId"); + String originServiceId = !StringUtils.isEmpty(tmp) ? tmp : null; tmp = msg.getMetaData().getValue("expirationTime"); long expirationTime = !StringUtils.isEmpty(tmp) ? Long.parseLong(tmp) : (System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(config.getTimeoutInSeconds())); @@ -106,24 +101,25 @@ public class TbSendRPCRequestNode implements TbNode { .oneway(oneway) .method(json.get("method").getAsString()) .body(params) + .tenantId(ctx.getTenantId()) .deviceId(new DeviceId(msg.getOriginator().getId())) .requestId(requestId) .requestUUID(requestUUID) - .originHost(originHost) - .originPort(originPort) + .originServiceId(originServiceId) .expirationTime(expirationTime) .restApiCall(restApiCall) .build(); - ctx.getRpcService().sendRpcRequest(request, ruleEngineDeviceRpcResponse -> { + ctx.getRpcService().sendRpcRequestToDevice(request, ruleEngineDeviceRpcResponse -> { if (!ruleEngineDeviceRpcResponse.getError().isPresent()) { - TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); + ctx.enqueueForTellNext(next, TbRelationTypes.SUCCESS); } else { - TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); - ctx.tellFailure(next, new RuntimeException(ruleEngineDeviceRpcResponse.getError().get().name())); + TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); + ctx.enqueueForTellFailure(next, ruleEngineDeviceRpcResponse.getError().get().name()); } }); + ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java index d01f28f136..bd3a3cdfb2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java @@ -16,18 +16,40 @@ package org.thingsboard.rule.engine.rpc; import lombok.Data; +import org.springframework.util.StringUtils; import org.thingsboard.rule.engine.api.NodeConfiguration; import org.thingsboard.server.common.data.DataConstants; @Data public class TbSendRpcReplyNodeConfiguration implements NodeConfiguration { + public static final String SERVICE_ID = "serviceId"; + public static final String SESSION_ID = "sessionId"; + public static final String REQUEST_ID = "requestId"; + + private String serviceIdMetaDataAttribute; + private String sessionIdMetaDataAttribute; private String requestIdMetaDataAttribute; @Override public TbSendRpcReplyNodeConfiguration defaultConfiguration() { TbSendRpcReplyNodeConfiguration configuration = new TbSendRpcReplyNodeConfiguration(); - configuration.setRequestIdMetaDataAttribute("requestId"); + configuration.setServiceIdMetaDataAttribute(SERVICE_ID); + configuration.setSessionIdMetaDataAttribute(SESSION_ID); + configuration.setRequestIdMetaDataAttribute(REQUEST_ID); return configuration; } + + public String getServiceIdMetaDataAttribute() { + return !StringUtils.isEmpty(serviceIdMetaDataAttribute) ? serviceIdMetaDataAttribute : SERVICE_ID; + } + + public String getSessionIdMetaDataAttribute() { + return !StringUtils.isEmpty(sessionIdMetaDataAttribute) ? sessionIdMetaDataAttribute : SESSION_ID; + } + + public String getRequestIdMetaDataAttribute() { + return !StringUtils.isEmpty(requestIdMetaDataAttribute) ? requestIdMetaDataAttribute : REQUEST_ID; + } } + diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 46a68d3e26..8d80e17f87 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -61,13 +61,9 @@ public class TbMsgAttributesNode implements TbNode { ctx.tellFailure(msg, new IllegalArgumentException("Unsupported msg type: " + msg.getType())); return; } - String src = msg.getData(); Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(src)); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), config.getScope(), new ArrayList<>(attributes), new TelemetryNodeCallback(ctx, msg)); - if (msg.getOriginator().getEntityType() == EntityType.DEVICE && DataConstants.SHARED_SCOPE.equals(config.getScope())) { - ctx.getTelemetryService().onSharedAttributesUpdate(ctx.getTenantId(), new DeviceId(msg.getOriginator().getId()), attributes); - } } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index ce7edf1c5f..14565ac475 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -74,7 +74,7 @@ public class TbMsgTimeseriesNode implements TbNode { } String src = msg.getData(); Map> tsKvMap = JsonConverter.convertToTelemetry(new JsonParser().parse(src), ts); - if (tsKvMap == null) { + if (tsKvMap.isEmpty()) { ctx.tellFailure(msg, new IllegalArgumentException("Msg body is empty: " + src)); return; } @@ -85,7 +85,7 @@ public class TbMsgTimeseriesNode implements TbNode { } } String ttlValue = msg.getMetaData().getValue("TTL"); - long ttl = !StringUtils.isEmpty(ttlValue) ? Long.valueOf(ttlValue) : config.getDefaultTTL(); + long ttl = !StringUtils.isEmpty(ttlValue) ? Long.parseLong(ttlValue) : config.getDefaultTTL(); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), tsKvEntryList, ttl, new TelemetryNodeCallback(ctx, msg)); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java index d545d41d7b..bd15c5bd83 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java @@ -22,8 +22,6 @@ import org.thingsboard.server.common.msg.TbMsg; import javax.annotation.Nullable; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; - /** * Created by ashvayka on 02.04.18. */ @@ -34,7 +32,7 @@ class TelemetryNodeCallback implements FutureCallback { @Override public void onSuccess(@Nullable Void result) { - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index 5423236499..31501688eb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -25,10 +25,6 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgTransactionData; - -import java.util.concurrent.ExecutionException; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @@ -43,31 +39,17 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; "Size of the queue per originator and timeout values are configurable on a system level", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbNodeEmptyConfig") +@Deprecated public class TbSynchronizationBeginNode implements TbNode { - private EmptyNodeConfiguration config; - @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - log.trace("Msg enters transaction - [{}][{}]", msg.getId(), msg.getType()); - - TbMsgTransactionData transactionData = new TbMsgTransactionData(msg.getId(), msg.getOriginator()); - TbMsg tbMsg = new TbMsg(msg.getId(), msg.getType(), msg.getOriginator(), msg.getMetaData(), TbMsgDataType.JSON, - msg.getData(), transactionData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); - - ctx.getRuleChainTransactionService().beginTransaction(tbMsg, startMsg -> { - log.trace("Transaction starting...[{}][{}]", startMsg.getId(), startMsg.getType()); - ctx.tellNext(startMsg, SUCCESS); - }, endMsg -> log.trace("Transaction ended successfully...[{}][{}]", endMsg.getId(), endMsg.getType()), - throwable -> { - log.trace("Transaction failed! [{}][{}]", tbMsg.getId(), tbMsg.getType(), throwable); - ctx.tellFailure(tbMsg, throwable); - }); + log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java index af3742a0fe..ac3d42ef36 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java @@ -40,25 +40,20 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = ("tbNodeEmptyConfig") ) +@Deprecated public class TbSynchronizationEndNode implements TbNode { - private EmptyNodeConfiguration config; - @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - ctx.getRuleChainTransactionService().endTransaction(msg, - successMsg -> ctx.tellNext(successMsg, SUCCESS), - throwable -> ctx.tellFailure(msg, throwable)); - log.trace("Msg left transaction - [{}][{}]", msg.getId(), msg.getType()); + log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); + ctx.tellSuccess(msg); } @Override public void destroy() { - } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java index e768f699cd..84a6b76920 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java @@ -55,7 +55,7 @@ public abstract class TbAbstractTransformNode implements TbNode { protected void transformSuccess(TbContext ctx, TbMsg msg, TbMsg m) { if (m != null) { - ctx.tellNext(m, SUCCESS); + ctx.tellSuccess(m); } else { ctx.tellNext(msg, FAILURE); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java index 40a3e8effd..31fd207c7c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; public class EntitiesAlarmOriginatorIdAsyncLoader { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java index a0a1c8629f..182a000b59 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java @@ -22,7 +22,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.EntityFieldsData; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java index 3ff25e1e8b..017bacfd72 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; diff --git a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js index fd5d9b7c62..f848a91875 100644 --- a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js +++ b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js @@ -1,6 +1,6 @@ -!function(e){function t(a){if(n[a])return n[a].exports;var i=n[a]={exports:{},id:a,loaded:!1};return e[a].call(i.exports,i,i.exports,t),i.loaded=!0,i.exports}var n={};return t.m=e,t.c=n,t.p="/static/",t(0)}(function(e){for(var t in e)if(Object.prototype.hasOwnProperty.call(e,t))switch(typeof e[t]){case"function":break;case"object":e[t]=function(t){var n=t.slice(1),a=e[t[0]];return function(e,t,i){a.apply(this,[e,t,i].concat(n))}}(e[t]);break;default:e[t]=e[e[t]]}return e}([function(e,t,n){e.exports=n(103)},function(e,t){},1,1,1,1,function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
 {{ 'tb.rulenode.create-customer-if-not-exists' | translate }}
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports='
{{scope.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
{{ 'tb.rulenode.use-message-alarm-data' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
 {{ severity.name | translate}}
tb.rulenode.alarm-severity-required
{{ 'tb.rulenode.propagate' | translate }}
tb.rulenode.relation-types-list-hint
"},function(e,t){e.exports="
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.entity-type-pattern-required
tb.rulenode.entity-type-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
 {{ 'tb.rulenode.create-entity-if-not-exists' | translate }}
tb.rulenode.create-entity-if-not-exists-hint
{{ 'tb.rulenode.remove-current-relations' | translate }}
tb.rulenode.remove-current-relations-hint
{{ 'tb.rulenode.change-originator-to-related-entity' | translate }}
tb.rulenode.change-originator-to-related-entity-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.delete-relation-to-specific-entity' | translate }}
tb.rulenode.delete-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
tb.rulenode.message-count-required
tb.rulenode.min-message-count-message
tb.rulenode.period-seconds-required
tb.rulenode.min-period-seconds-message
{{ 'tb.rulenode.test-generator-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
tb.rulenode.min-inside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.min-outside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
'},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.bootstrap-servers-required
tb.rulenode.min-retries-message
tb.rulenode.min-batch-size-bytes-message
tb.rulenode.min-linger-ms-message
tb.rulenode.min-buffer-memory-bytes-message
 {{ ackValue }}
tb.rulenode.key-serializer-required
tb.rulenode.value-serializer-required
 {{ \'tb.rulenode.add-metadata-key-values-as-kafka-headers\' | translate }}
tb.rulenode.add-metadata-key-values-as-kafka-headers-hint
{{charset.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-to-string-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.mqtt-topic-pattern-hint
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
tb.rulenode.connect-timeout-required
tb.rulenode.connect-timeout-range
tb.rulenode.connect-timeout-range
{{ \'tb.rulenode.clean-session\' | translate }} {{ \'tb.rulenode.enable-ssl\' | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{credentialsValue.name | translate}}
tb.rulenode.credentials-type-required
tb.rulenode.username-required
tb.rulenode.password-required
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.caCertFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.certFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.privateKeyFileName}}
'; +!function(e){function t(a){if(n[a])return n[a].exports;var i=n[a]={exports:{},id:a,loaded:!1};return e[a].call(i.exports,i,i.exports,t),i.loaded=!0,i.exports}var n={};return t.m=e,t.c=n,t.p="/static/",t(0)}(function(e){for(var t in e)if(Object.prototype.hasOwnProperty.call(e,t))switch(typeof e[t]){case"function":break;case"object":e[t]=function(t){var n=t.slice(1),a=e[t[0]];return function(e,t,i){a.apply(this,[e,t,i].concat(n))}}(e[t]);break;default:e[t]=e[e[t]]}return e}([function(e,t,n){e.exports=n(105)},function(e,t){},1,1,1,1,function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
 {{ 'tb.rulenode.create-customer-if-not-exists' | translate }}
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports='
{{scope.name | translate}}
'},function(e,t){e.exports="
tb.rulenode.select-queue-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
{{ 'tb.rulenode.use-message-alarm-data' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
 {{ severity.name | translate}}
tb.rulenode.alarm-severity-required
{{ 'tb.rulenode.propagate' | translate }}
tb.rulenode.relation-types-list-hint
"},function(e,t){e.exports="
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.entity-type-pattern-required
tb.rulenode.entity-type-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
 {{ 'tb.rulenode.create-entity-if-not-exists' | translate }}
tb.rulenode.create-entity-if-not-exists-hint
{{ 'tb.rulenode.remove-current-relations' | translate }}
tb.rulenode.remove-current-relations-hint
{{ 'tb.rulenode.change-originator-to-related-entity' | translate }}
tb.rulenode.change-originator-to-related-entity-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.delete-relation-to-specific-entity' | translate }}
tb.rulenode.delete-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
tb.rulenode.message-count-required
tb.rulenode.min-message-count-message
tb.rulenode.period-seconds-required
tb.rulenode.min-period-seconds-message
{{ 'tb.rulenode.test-generator-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
tb.rulenode.min-inside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.min-outside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
'},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.bootstrap-servers-required
tb.rulenode.min-retries-message
tb.rulenode.min-batch-size-bytes-message
tb.rulenode.min-linger-ms-message
tb.rulenode.min-buffer-memory-bytes-message
 {{ ackValue }}
tb.rulenode.key-serializer-required
tb.rulenode.value-serializer-required
 {{ \'tb.rulenode.add-metadata-key-values-as-kafka-headers\' | translate }}
tb.rulenode.add-metadata-key-values-as-kafka-headers-hint
{{charset.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-to-string-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.mqtt-topic-pattern-hint
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
tb.rulenode.connect-timeout-required
tb.rulenode.connect-timeout-range
tb.rulenode.connect-timeout-range
{{ \'tb.rulenode.clean-session\' | translate }} {{ \'tb.rulenode.enable-ssl\' | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{credentialsValue.name | translate}}
tb.rulenode.credentials-type-required
tb.rulenode.username-required
tb.rulenode.password-required
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.caCertFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.certFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.privateKeyFileName}}
'; },function(e,t){e.exports="
tb.rulenode.interval-seconds-required
tb.rulenode.min-interval-seconds-message
tb.rulenode.output-timeseries-key-prefix-required
"},function(e,t){e.exports='
{{ \'tb.rulenode.use-metadata-period-in-seconds-patterns\' | translate }}
tb.rulenode.use-metadata-period-in-seconds-patterns-hint
tb.rulenode.period-seconds-required
tb.rulenode.min-period-0-seconds-message
tb.rulenode.period-in-seconds-pattern-required
tb.rulenode.period-in-seconds-pattern-hint
tb.rulenode.max-pending-messages-required
tb.rulenode.max-pending-messages-range
tb.rulenode.max-pending-messages-range
'},function(e,t){e.exports="
tb.rulenode.gcp-project-id-required
tb.rulenode.pubsub-topic-name-required
{{ 'action.remove' | translate }} close
tb.rulenode.no-file
{{configuration.serviceAccountKeyFileName}}
tb.rulenode.message-attributes-hint
"},function(e,t){e.exports='
{{ property }}
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
{{ \'tb.rulenode.automatic-recovery\' | translate }}
tb.rulenode.min-connection-timeout-ms-message
tb.rulenode.min-handshake-timeout-ms-message
'},function(e,t){e.exports='
tb.rulenode.endpoint-url-pattern-required
tb.rulenode.endpoint-url-pattern-hint
 {{ type }} {{ \'tb.rulenode.use-simple-client-http-factory\' | translate }}
tb.rulenode.read-timeout-hint
tb.rulenode.max-parallel-requests-count-hint
tb.rulenode.headers-hint
{{ \'tb.rulenode.use-redis-queue\' | translate }}
{{ \'tb.rulenode.trim-redis-queue\' | translate }}
'},function(e,t){e.exports="
"},function(e,t){e.exports="
tb.rulenode.timeout-required
tb.rulenode.min-timeout-message
"},function(e,t){e.exports='
tb.rulenode.custom-table-name-required
tb.rulenode.custom-table-hint
'},function(e,t){e.exports='
{{ \'tb.rulenode.use-system-smtp-settings\' | translate }}
{{smtpProtocol.toUpperCase()}}
tb.rulenode.smtp-host-required
tb.rulenode.smtp-port-required
tb.rulenode.smtp-port-range
tb.rulenode.smtp-port-range
tb.rulenode.timeout-required
tb.rulenode.min-timeout-msec-message
 {{ \'tb.rulenode.enable-tls\' | translate }} {{tlsVersion}}
'},function(e,t){e.exports="
tb.rulenode.topic-arn-pattern-required
tb.rulenode.topic-arn-pattern-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
"},function(e,t){e.exports='
{{ type.name | translate }}
tb.rulenode.queue-url-pattern-required
tb.rulenode.queue-url-pattern-hint
tb.rulenode.min-delay-seconds-message
tb.rulenode.max-delay-seconds-message
tb.rulenode.message-attributes-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
'},function(e,t){e.exports="
tb.rulenode.default-ttl-required
tb.rulenode.min-default-ttl-message
"},function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-type
device.device-types
"},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
{{\'tb.rulenode.entity-details-\'+item.toLowerCase() | translate}} tb.rulenode.no-entity-details-matching {{\'tb.rulenode.entity-details-\'+$chip.toLowerCase() | translate}} {{ \'tb.rulenode.add-to-metadata\' | translate }}
tb.rulenode.add-to-metadata-hint
'},function(e,t){e.exports='
{{ type }}
tb.rulenode.fetch-mode-hint
{{ type }}
tb.rulenode.order-by-hint
tb.rulenode.limit-hint
{{ \'tb.rulenode.use-metadata-interval-patterns\' | translate }}
tb.rulenode.use-metadata-interval-patterns-hint
tb.rulenode.start-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.end-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.start-interval-pattern-required
tb.rulenode.start-interval-pattern-hint
tb.rulenode.end-interval-pattern-required
tb.rulenode.end-interval-pattern-hint
'; -},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},31,function(e,t){e.exports="
{{'alarm.display-status.' + item | translate}} {{'alarm.display-status.' + $chip | translate}}
"},function(e,t){e.exports='
tb.rulenode.separator-hint
tb.rulenode.separator-hint
{{ \'tb.rulenode.check-all-keys\' | translate }}
tb.rulenode.check-all-keys-hint
'},function(e,t){e.exports="
{{ 'tb.rulenode.check-relation-to-specific-entity' | translate }}
tb.rulenode.check-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
'},function(e,t){e.exports='
{{item}}
tb.rulenode.no-message-types-found
tb.rulenode.no-message-type-matching tb.rulenode.create-new-message-type
{{$chip.name}}
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-filter-function' | translate }}
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-switch-function' | translate }}
"},function(e,t){e.exports='
{{ keyText }} {{ valText }}  
{{keyRequiredText}}
{{valRequiredText}}
 {{ \'tb.key-val.remove-entry\' | translate }} close
{{ \'tb.key-val.add-entry\' | translate }} add {{ \'action.add\' | translate }}
'},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-filters
"},function(e,t){e.exports='
{{ source.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-transformer-function' | translate }}
"},function(e,t){e.exports="
tb.rulenode.from-template-required
tb.rulenode.from-template-hint
tb.rulenode.to-template-required
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.subject-template-required
tb.rulenode.subject-template-hint
tb.rulenode.body-template-required
tb.rulenode.body-template-hint
"},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(6),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(7),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(8),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.hasOwnProperty("relationTypes")||(i.configuration.relationTypes=[])},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(9),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(10),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(11),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.originator=null,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.originatorId&&i.configuration.originatorType?i.originator={id:i.configuration.originatorId,entityType:i.configuration.originatorType}:i.originator=null,i.$watch("originator",function(e,t){angular.equals(e,t)||(i.originator?(s.$viewValue.originatorId=i.originator.id,s.$viewValue.originatorType=i.originator.entityType):(s.$viewValue.originatorId=null,s.$viewValue.originatorType=null))},!0)},i.testScript=function(e){var n=angular.copy(i.configuration.jsScript);a.testNodeScript(e,n,"generate",t.instant("tb.rulenode.generator")+"","Generate",["prevMsg","prevMetadata","prevMsgType"],i.ruleNodeId).then(function(e){i.configuration.jsScript=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(1);var r=n(12),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(13),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(75),r=a(i),o=n(53),l=a(o),s=n(58),d=a(s),u=n(55),c=a(u),m=n(54),g=a(m),p=n(62),f=a(p),b=n(69),v=a(b),y=n(70),h=a(y),q=n(68),k=a(q),x=n(61),$=a(x),T=n(73),C=a(T),w=n(74),M=a(w),S=n(67),N=a(S),_=n(63),F=a(_),E=n(72),P=a(E),A=n(65),V=a(A),I=n(64),O=a(I),j=n(52),D=a(j),L=n(76),R=a(L),K=n(57),U=a(K),z=n(56),H=a(z),B=n(71),G=a(B),Y=n(59),Q=a(Y),W=n(66),J=a(W);t.default=angular.module("thingsboard.ruleChain.config.action",[]).directive("tbActionNodeTimeseriesConfig",r.default).directive("tbActionNodeAttributesConfig",l.default).directive("tbActionNodeGeneratorConfig",d.default).directive("tbActionNodeCreateAlarmConfig",c.default).directive("tbActionNodeClearAlarmConfig",g.default).directive("tbActionNodeLogConfig",f.default).directive("tbActionNodeRpcReplyConfig",v.default).directive("tbActionNodeRpcRequestConfig",h.default).directive("tbActionNodeRestApiCallConfig",k.default).directive("tbActionNodeKafkaConfig",$.default).directive("tbActionNodeSnsConfig",C.default).directive("tbActionNodeSqsConfig",M.default).directive("tbActionNodeRabbitMqConfig",N.default).directive("tbActionNodeMqttConfig",F.default).directive("tbActionNodeSendEmailConfig",P.default).directive("tbActionNodeMsgDelayConfig",V.default).directive("tbActionNodeMsgCountConfig",O.default).directive("tbActionNodeAssignToCustomerConfig",D.default).directive("tbActionNodeUnAssignToCustomerConfig",R.default).directive("tbActionNodeDeleteRelationConfig",U.default).directive("tbActionNodeCreateRelationConfig",H.default).directive("tbActionNodeCustomTableConfig",G.default).directive("tbActionNodeGpsGeofencingConfig",Q.default).directive("tbActionNodePubSubConfig",J.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ackValues=["all","-1","0","1"],n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue,n.configuration.hasOwnProperty("kafkaHeadersCharset")||(n.configuration.kafkaHeadersCharset="UTF-8")},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(14),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"string",t.instant("tb.rulenode.to-string")+"","ToString",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(15),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$mdExpansionPanel=t,a.ruleNodeTypes=n,a.credentialsTypeChanged=function(){var e=a.configuration.credentials.type;a.configuration.credentials={},a.configuration.credentials.type=e,a.updateValidity()},a.certFileAdded=function(e,t){var n=new FileReader;n.onload=function(n){a.$apply(function(){if(n.target.result){l.$setDirty();var i=n.target.result;i&&i.length>0&&("caCert"==t&&(a.configuration.credentials.caCertFileName=e.name,a.configuration.credentials.caCert=i),"privateKey"==t&&(a.configuration.credentials.privateKeyFileName=e.name,a.configuration.credentials.privateKey=i),"Cert"==t&&(a.configuration.credentials.certFileName=e.name,a.configuration.credentials.cert=i)),a.updateValidity()}})},n.readAsText(e.file)},a.clearCertFile=function(e){l.$setDirty(),"caCert"==e&&(a.configuration.credentials.caCertFileName=null,a.configuration.credentials.caCert=null),"privateKey"==e&&(a.configuration.credentials.privateKeyFileName=null,a.configuration.credentials.privateKey=null),"Cert"==e&&(a.configuration.credentials.certFileName=null,a.configuration.credentials.cert=null),a.updateValidity()},a.updateValidity=function(){var e=!0,t=a.configuration.credentials;t.type==n.mqttCredentialTypes["cert.PEM"].value&&(t.caCert&&t.cert&&t.privateKey||(e=!1)),l.$setValidity("Certs",e)},a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:a}}i.$inject=["$compile","$mdExpansionPanel","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(2);var r=n(16),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(17),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(18),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.serviceAccountFileAdded=function(e){var t=new FileReader; -t.onload=function(t){n.$apply(function(){if(t.target.result){r.$setDirty();var a=t.target.result;a&&a.length>0&&(n.configuration.serviceAccountKeyFileName=e.name,n.configuration.serviceAccountKey=a),n.updateValidity()}})},t.readAsText(e.file)},n.clearServiceAccountFile=function(){r.$setDirty(),n.configuration.serviceAccountKeyFileName=null,n.configuration.serviceAccountKey=null,n.updateValidity()},n.updateValidity=function(){var e=!0,t=n.configuration;t.serviceAccountKeyFileName&&t.serviceAccountKey||(e=!1),r.$setValidity("SAKey",e)},n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(19),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(20),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(21),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(22),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(23),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(24),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.smtpProtocols=["smtp","smtps"],t.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(25),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(26),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(27),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(28),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(29),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(30),o=a(r)},function(e,t){"use strict";function n(e){var t=function(t,n,a,i){n.html("
"),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}n.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(31),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(32),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.entityDetailsList=[];for(var s in t.entityDetails){var d=s;n.entityDetailsList.push(d)}r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(33),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s);var d=186;a.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,d],a.ruleNodeTypes=n,a.aggPeriodTimeUnits={},a.aggPeriodTimeUnits.MINUTES=n.timeUnit.MINUTES,a.aggPeriodTimeUnits.HOURS=n.timeUnit.HOURS,a.aggPeriodTimeUnits.DAYS=n.timeUnit.DAYS,a.aggPeriodTimeUnits.MILLISECONDS=n.timeUnit.MILLISECONDS,a.aggPeriodTimeUnits.SECONDS=n.timeUnit.SECONDS,a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{},link:a}}i.$inject=["$compile","$mdConstant","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(34),o=a(r);n(3)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(84),r=a(i),o=n(85),l=a(o),s=n(80),d=a(s),u=n(86),c=a(u),m=n(79),g=a(m),p=n(87),f=a(p),b=n(82),v=a(b),y=n(81),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.enrichment",[]).directive("tbEnrichmentNodeOriginatorAttributesConfig",r.default).directive("tbEnrichmentNodeOriginatorFieldsConfig",l.default).directive("tbEnrichmentNodeDeviceAttributesConfig",d.default).directive("tbEnrichmentNodeRelatedAttributesConfig",c.default).directive("tbEnrichmentNodeCustomerAttributesConfig",g.default).directive("tbEnrichmentNodeTenantAttributesConfig",f.default).directive("tbEnrichmentNodeGetTelemetryFromDatabase",v.default).directive("tbEnrichmentNodeEntityDetailsConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(35),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(36),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(37),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(38),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.alarmStatusList=[];for(var s in t.alarmStatus)n.alarmStatusList.push(t.alarmStatus[s]);r.$render=function(){n.configuration=r.$viewValue},n.getAlarmStatusList=function(){return n.alarmStatusList.filter(function(e){return n.configuration.alarmStatusList.indexOf(e)===-1})},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(39),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(40),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(41),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(42),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(95),r=a(i),o=n(93),l=a(o),s=n(96),d=a(s),u=n(90),c=a(u),m=n(94),g=a(m),p=n(89),f=a(p),b=n(91),v=a(b),y=n(88),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.filter",[]).directive("tbFilterNodeScriptConfig",r.default).directive("tbFilterNodeMessageTypeConfig",l.default).directive("tbFilterNodeSwitchConfig",d.default).directive("tbFilterNodeCheckRelationConfig",c.default).directive("tbFilterNodeOriginatorTypeConfig",g.default).directive("tbFilterNodeCheckMessageConfig",f.default).directive("tbFilterNodeGpsGeofencingConfig",v.default).directive("tbFilterNodeCheckAlarmStatusConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){function s(){if(l.$viewValue){for(var e=[],t=0;t-1&&t.kvList.splice(e,1)}function l(){t.kvList||(t.kvList=[]),t.kvList.push({key:"",value:""})}function s(){var e={};t.kvList.forEach(function(t){t.key&&(e[t.key]=t.value)}),i.$setViewValue(e),d()}function d(){var e=!0;t.required&&!t.kvList.length&&(e=!1),i.$setValidity("kvMap",e)}var u=o.default;n.html(u),t.ngModelCtrl=i,t.removeKeyVal=r,t.addKeyVal=l,t.kvList=[],t.$watch("query",function(e,n){angular.equals(e,n)||i.$setViewValue(t.query)}),i.$render=function(){if(i.$viewValue){var e=i.$viewValue;t.kvList.length=0;for(var n in e)t.kvList.push({key:n,value:e[n]})}t.$watch("kvList",function(e,t){angular.equals(e,t)||s()},!0),d()},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",disabled:"=ngDisabled",requiredText:"=",keyText:"=",keyRequiredText:"=",valText:"=",valRequiredText:"="},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(47),o=a(r);n(5)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(48),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(49),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(99),r=a(i),o=n(101),l=a(o),s=n(102),d=a(s);t.default=angular.module("thingsboard.ruleChain.config.transform",[]).directive("tbTransformationNodeChangeOriginatorConfig",r.default).directive("tbTransformationNodeScriptConfig",l.default).directive("tbTransformationNodeToEmailConfig",d.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"update",t.instant("tb.rulenode.transformer")+"","Transform",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(50),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(51),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(106),r=a(i),o=n(92),l=a(o),s=n(83),d=a(s),u=n(100),c=a(u),m=n(60),g=a(m),p=n(78),f=a(p),b=n(98),v=a(b),y=n(77),h=a(y),q=n(97),k=a(q),x=n(105),$=a(x);t.default=angular.module("thingsboard.ruleChain.config",[r.default,l.default,d.default,c.default,g.default]).directive("tbNodeEmptyConfig",f.default).directive("tbRelationsQueryConfig",v.default).directive("tbDeviceRelationsQueryConfig",h.default).directive("tbKvMapConfig",k.default).config($.default).name},function(e,t){"use strict";function n(e){var t={tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647'.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-statuses-filter":"Alarm statuses filter","alarm-statuses-required":"Alarm statuses is required",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required","endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata", -header:"Header","header-required":"Header is required",value:"Value","value-required":"Value is required","topic-pattern":"Topic pattern","topic-pattern-required":"Topic pattern is required","mqtt-topic-pattern-hint":"MQTT topic pattern, use ${metaKeyName} to substitute variables from metadata","bootstrap-servers":"Bootstrap servers","bootstrap-servers-required":"Bootstrap servers value is required","other-properties":"Other properties",key:"Key","key-required":"Key is required",retries:"Automatically retry times if fails","min-retries-message":"Only 0 minimum retries is allowed.","batch-size-bytes":"Produces batch size in bytes","min-batch-size-bytes-message":"Only 0 minimum batch size is allowed.","linger-ms":"Time to buffer locally (ms)","min-linger-ms-message":"Only 0 ms minimum value is allowed.","buffer-memory-bytes":"Client buffer max size in bytes","min-buffer-memory-message":"Only 0 minimum buffer size is allowed.",acks:"Number of acknowledgments","key-serializer":"Key serializer","key-serializer-required":"Key serializer is required","value-serializer":"Value serializer","value-serializer-required":"Value serializer is required","topic-arn-pattern":"Topic ARN pattern","topic-arn-pattern-required":"Topic ARN pattern is required","topic-arn-pattern-hint":"Topic ARN pattern, use ${metaKeyName} to substitute variables from metadata","aws-access-key-id":"AWS Access Key ID","aws-access-key-id-required":"AWS Access Key ID is required","aws-secret-access-key":"AWS Secret Access Key","aws-secret-access-key-required":"AWS Secret Access Key is required","aws-region":"AWS Region","aws-region-required":"AWS Region is required","exchange-name-pattern":"Exchange name pattern","routing-key-pattern":"Routing key pattern","message-properties":"Message properties",host:"Host","host-required":"Host is required",port:"Port","port-required":"Port is required","port-range":"Port should be in a range from 1 to 65535.","virtual-host":"Virtual host",username:"Username",password:"Password","automatic-recovery":"Automatic recovery","connection-timeout-ms":"Connection timeout (ms)","min-connection-timeout-ms-message":"Only 0 ms minimum value is allowed.","handshake-timeout-ms":"Handshake timeout (ms)","min-handshake-timeout-ms-message":"Only 0 ms minimum value is allowed.","client-properties":"Client properties","queue-url-pattern":"Queue URL pattern","queue-url-pattern-required":"Queue URL pattern is required","queue-url-pattern-hint":"Queue URL pattern, use ${metaKeyName} to substitute variables from metadata","delay-seconds":"Delay (seconds)","min-delay-seconds-message":"Only 0 seconds minimum value is allowed.","max-delay-seconds-message":"Only 900 seconds maximum value is allowed.",name:"Name","name-required":"Name is required","queue-type":"Queue type","sqs-queue-standard":"Standard","sqs-queue-fifo":"FIFO","gcp-project-id":"GCP project ID","gcp-project-id-required":"GCP project ID is required","gcp-service-account-key":"GCP service account key file","gcp-service-account-key-required":"GCP service account key file is required","pubsub-topic-name":"Topic name","pubsub-topic-name-required":"Topic name is required","message-attributes":"Message attributes","message-attributes-hint":"Use ${metaKeyName} in name/value fields to substitute variables from metadata","connect-timeout":"Connection timeout (sec)","connect-timeout-required":"Connection timeout is required.","connect-timeout-range":"Connection timeout should be in a range from 1 to 200.","client-id":"Client ID","clean-session":"Clean session","enable-ssl":"Enable SSL",credentials:"Credentials","credentials-type":"Credentials type","credentials-type-required":"Credentials type is required.","credentials-anonymous":"Anonymous","credentials-basic":"Basic","credentials-pem":"PEM","username-required":"Username is required.","password-required":"Password is required.","ca-cert":"CA certificate file *","private-key":"Private key file *",cert:"Certificate file *","no-file":"No file selected.","drop-file":"Drop a file or click to select a file to upload.","private-key-password":"Private key password","use-system-smtp-settings":"Use system SMTP settings","use-metadata-interval-patterns":"Use metadata interval patterns","use-metadata-interval-patterns-hint":"If selected, rule node use start and end interval patterns from message metadata assuming that intervals are in the milliseconds.","use-message-alarm-data":"Use message alarm data","check-all-keys":"Check that all selected keys are present","check-all-keys-hint":"If selected, checks that all specified keys are present in the message data and metadata.","check-relation-to-specific-entity":"Check relation to specific entity","check-relation-hint":"Checks existence of relation to specific entity or to any entity based on direction and relation type.","delete-relation-to-specific-entity":"Delete relation to specific entity","delete-relation-hint":"Deletes relation from the originator of the incoming message to the specified entity or list of entities based on direction and type.","remove-current-relations":"Remove current relations","remove-current-relations-hint":"Removes current relations from the originator of the incoming message based on direction and type.","change-originator-to-related-entity":"Change originator to related entity","change-originator-to-related-entity-hint":"Used to process submitted message as a message from another entity.","start-interval-pattern":"Start interval pattern","end-interval-pattern":"End interval pattern","start-interval-pattern-required":"Start interval pattern is required","end-interval-pattern-required":"End interval pattern is required","start-interval-pattern-hint":"Start interval pattern, use ${metaKeyName} to substitute variables from metadata","end-interval-pattern-hint":"End interval pattern, use ${metaKeyName} to substitute variables from metadata","smtp-protocol":"Protocol","smtp-host":"SMTP host","smtp-host-required":"SMTP host is required.","smtp-port":"SMTP port","smtp-port-required":"You must supply a smtp port.","smtp-port-range":"SMTP port should be in a range from 1 to 65535.","timeout-msec":"Timeout ms","min-timeout-msec-message":"Only 0 ms minimum value is allowed.","enter-username":"Enter username","enter-password":"Enter password","enable-tls":"Enable TLS","tls-version":"TLS version","min-period-0-seconds-message":"Only 0 second minimum period is allowed.","max-pending-messages":"Maximum pending messages","max-pending-messages-required":"Maximum pending messages is required.","max-pending-messages-range":"Maximum pending messages should be in a range from 1 to 100000.","originator-types-filter":"Originator types filter","interval-seconds":"Interval in seconds","interval-seconds-required":"Interval is required.","min-interval-seconds-message":"Only 1 second minimum interval is allowed.","output-timeseries-key-prefix":"Output timeseries key prefix","output-timeseries-key-prefix-required":"Output timeseries key prefix required.","separator-hint":'You should press "enter" to complete field input.',"entity-details":"Select entity details:","entity-details-title":"Title","entity-details-country":"Country","entity-details-state":"State","entity-details-zip":"Zip","entity-details-address":"Address","entity-details-address2":"Address2","entity-details-additional_info":"Additional Info","entity-details-phone":"Phone","entity-details-email":"Email","add-to-metadata":"Add selected details to message metadata","add-to-metadata-hint":"If selected, adds the selected details keys to the message metadata instead of message data.","entity-details-list-empty":"No entity details selected.","no-entity-details-matching":"No entity details matching were found.","custom-table-name":"Custom table name","custom-table-name-required":"Table Name is required","custom-table-hint":"You should enter the table name without prefix 'cs_tb_'.","message-field":"Message field","message-field-required":"Message field is required.","table-col":"Table column","table-col-required":"Table column is required.","latitude-key-name":"Latitude key name","longitude-key-name":"Longitude key name","latitude-key-name-required":"Latitude key name is required.","longitude-key-name-required":"Longitude key name is required.","fetch-perimeter-info-from-message-metadata":"Fetch perimeter information from message metadata","perimeter-circle":"Circle","perimeter-polygon":"Polygon","perimeter-type":"Perimeter type","circle-center-latitude":"Center latitude","circle-center-latitude-required":"Center latitude is required.","circle-center-longitude":"Center longitude","circle-center-longitude-required":"Center longitude is required.","range-unit-meter":"Meter","range-unit-kilometer":"Kilometer","range-unit-foot":"Foot","range-unit-mile":"Mile","range-unit-nautical-mile":"Nautical mile","range-units":"Range units",range:"Range","range-required":"Range is required.","polygon-definition":"Polygon definition","polygon-definition-required":"Polygon definition is required.","polygon-definition-hint":"Please, use the following format for manual definition of polygon: [[lat1,lon1],[lat2,lon2], ... ,[latN,lonN]].","min-inside-duration":"Minimal inside duration","min-inside-duration-value-required":"Minimal inside duration is required","min-inside-duration-time-unit":"Minimal inside duration time unit","min-outside-duration":"Minimal outside duration","min-outside-duration-value-required":"Minimal outside duration is required","min-outside-duration-time-unit":"Minimal outside duration time unit","tell-failure-if-absent":"Tell Failure","tell-failure-if-absent-hint":'If at least one selected key doesn\'t exist the outbound message will report "Failure".',"get-latest-value-with-ts":"Fetch Latest telemetry with Timestamp","get-latest-value-with-ts-hint":'If selected, latest telemetry values will be added to the outbound message metadata with timestamp, e.g: "temp": "{\\"ts\\":1574329385897,\\"value\\":42}"',"use-redis-queue":"Use redis queue for message persistence","trim-redis-queue":"Trim redis queue","redis-queue-max-size":"Redis queue max size","add-metadata-key-values-as-kafka-headers":"Add Message metadata key-value pairs to Kafka record headers","add-metadata-key-values-as-kafka-headers-hint":"If selected, key-value pairs from message metadata will be added to the outgoing records headers as byte arrays with predefined charset encoding.","charset-encoding":"Charset encoding","charset-encoding-required":"Charset encoding is required.","charset-us-ascii":"US-ASCII","charset-iso-8859-1":"ISO-8859-1","charset-utf-8":"UTF-8","charset-utf-16be":"UTF-16BE","charset-utf-16le":"UTF-16LE","charset-utf-16":"UTF-16"},"key-val":{key:"Key",value:"Value","remove-entry":"Remove entry","add-entry":"Add entry"}}};e.translations("en_US",t)}Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){(0,o.default)(e)}i.$inject=["$translateProvider"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(104),o=a(r)},function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=angular.module("thingsboard.ruleChain.config.types",[]).constant("ruleNodeTypes",{originatorSource:{CUSTOMER:{name:"tb.rulenode.originator-customer",value:"CUSTOMER"},TENANT:{name:"tb.rulenode.originator-tenant",value:"TENANT"},RELATED:{name:"tb.rulenode.originator-related",value:"RELATED"},ALARM_ORIGINATOR:{name:"tb.rulenode.originator-alarm-originator",value:"ALARM_ORIGINATOR"}},fetchModeType:["FIRST","LAST","ALL"],samplingOrder:["ASC","DESC"],httpRequestType:["GET","POST","PUT","DELETE"],entityDetails:{TITLE:{name:"tb.rulenode.entity-details-title",value:"TITLE"},COUNTRY:{name:"tb.rulenode.entity-details-country",value:"COUNTRY"},STATE:{name:"tb.rulenode.entity-details-state",value:"STATE"},ZIP:{name:"tb.rulenode.entity-details-zip",value:"ZIP"},ADDRESS:{name:"tb.rulenode.entity-details-address",value:"ADDRESS"},ADDRESS2:{name:"tb.rulenode.entity-details-address2",value:"ADDRESS2"},PHONE:{name:"tb.rulenode.entity-details-phone",value:"PHONE"},EMAIL:{name:"tb.rulenode.entity-details-email",value:"EMAIL"},ADDITIONAL_INFO:{name:"tb.rulenode.entity-details-additional_info",value:"ADDITIONAL_INFO"}},sqsQueueType:{STANDARD:{name:"tb.rulenode.sqs-queue-standard",value:"STANDARD"},FIFO:{name:"tb.rulenode.sqs-queue-fifo",value:"FIFO"}},perimeterType:{CIRCLE:{name:"tb.rulenode.perimeter-circle",value:"CIRCLE"},POLYGON:{name:"tb.rulenode.perimeter-polygon",value:"POLYGON"}},timeUnit:{MILLISECONDS:{value:"MILLISECONDS",name:"tb.rulenode.time-unit-milliseconds"},SECONDS:{value:"SECONDS",name:"tb.rulenode.time-unit-seconds"},MINUTES:{value:"MINUTES",name:"tb.rulenode.time-unit-minutes"},HOURS:{value:"HOURS",name:"tb.rulenode.time-unit-hours"},DAYS:{value:"DAYS",name:"tb.rulenode.time-unit-days"}},rangeUnit:{METER:{value:"METER",name:"tb.rulenode.range-unit-meter"},KILOMETER:{value:"KILOMETER",name:"tb.rulenode.range-unit-kilometer"},FOOT:{value:"FOOT",name:"tb.rulenode.range-unit-foot"},MILE:{value:"MILE",name:"tb.rulenode.range-unit-mile"},NAUTICAL_MILE:{value:"NAUTICAL_MILE",name:"tb.rulenode.range-unit-nautical-mile"}},mqttCredentialTypes:{anonymous:{value:"anonymous",name:"tb.rulenode.credentials-anonymous"},basic:{value:"basic",name:"tb.rulenode.credentials-basic"},"cert.PEM":{value:"cert.PEM",name:"tb.rulenode.credentials-pem"}},toBytesStandartCharsetTypes:{"US-ASCII":{value:"US-ASCII",name:"tb.rulenode.charset-us-ascii"},"ISO-8859-1":{value:"ISO-8859-1",name:"tb.rulenode.charset-iso-8859-1"},"UTF-8":{value:"UTF-8",name:"tb.rulenode.charset-utf-8"},"UTF-16BE":{value:"UTF-16BE",name:"tb.rulenode.charset-utf-16be"},"UTF-16LE":{value:"UTF-16LE",name:"tb.rulenode.charset-utf-16le"},"UTF-16":{value:"UTF-16",name:"tb.rulenode.charset-utf-16"}}}).name}])); +},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},32,function(e,t){e.exports="
{{'alarm.display-status.' + item | translate}} {{'alarm.display-status.' + $chip | translate}}
"},function(e,t){e.exports='
tb.rulenode.separator-hint
tb.rulenode.separator-hint
{{ \'tb.rulenode.check-all-keys\' | translate }}
tb.rulenode.check-all-keys-hint
'},function(e,t){e.exports="
{{ 'tb.rulenode.check-relation-to-specific-entity' | translate }}
tb.rulenode.check-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
'},function(e,t){e.exports='
{{item}}
tb.rulenode.no-message-types-found
tb.rulenode.no-message-type-matching tb.rulenode.create-new-message-type
{{$chip.name}}
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-filter-function' | translate }}
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-switch-function' | translate }}
"},function(e,t){e.exports='
{{ keyText }} {{ valText }}  
{{keyRequiredText}}
{{valRequiredText}}
 {{ \'tb.key-val.remove-entry\' | translate }} close
{{ \'tb.key-val.add-entry\' | translate }} add {{ \'action.add\' | translate }}
'},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-filters
"},function(e,t){e.exports='
{{ source.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-transformer-function' | translate }}
"},function(e,t){e.exports="
tb.rulenode.from-template-required
tb.rulenode.from-template-hint
tb.rulenode.to-template-required
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.subject-template-required
tb.rulenode.subject-template-hint
tb.rulenode.body-template-required
tb.rulenode.body-template-hint
"},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(6),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(7),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.serviceType="TB_RULE_ENGINE",n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(8),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(9),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.hasOwnProperty("relationTypes")||(i.configuration.relationTypes=[])},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(10),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(11),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(12),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.originator=null,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.originatorId&&i.configuration.originatorType?i.originator={id:i.configuration.originatorId,entityType:i.configuration.originatorType}:i.originator=null,i.$watch("originator",function(e,t){angular.equals(e,t)||(i.originator?(s.$viewValue.originatorId=i.originator.id,s.$viewValue.originatorType=i.originator.entityType):(s.$viewValue.originatorId=null,s.$viewValue.originatorType=null))},!0)},i.testScript=function(e){var n=angular.copy(i.configuration.jsScript);a.testNodeScript(e,n,"generate",t.instant("tb.rulenode.generator")+"","Generate",["prevMsg","prevMetadata","prevMsgType"],i.ruleNodeId).then(function(e){i.configuration.jsScript=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(1);var r=n(13),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(14),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(77),r=a(i),o=n(54),l=a(o),s=n(60),d=a(s),u=n(57),c=a(u),m=n(56),g=a(m),p=n(64),f=a(p),b=n(71),v=a(b),y=n(72),h=a(y),q=n(70),k=a(q),x=n(63),$=a(x),T=n(75),C=a(T),w=n(76),M=a(w),N=n(69),S=a(N),_=n(65),F=a(_),E=n(74),P=a(E),A=n(67),V=a(A),I=n(66),j=a(I),O=n(53),D=a(O),L=n(78),R=a(L),K=n(59),U=a(K),z=n(58),H=a(z),B=n(73),G=a(B),Y=n(61),Q=a(Y),W=n(68),J=a(W),Z=n(55),X=a(Z);t.default=angular.module("thingsboard.ruleChain.config.action",[]).directive("tbActionNodeTimeseriesConfig",r.default).directive("tbActionNodeAttributesConfig",l.default).directive("tbActionNodeGeneratorConfig",d.default).directive("tbActionNodeCreateAlarmConfig",c.default).directive("tbActionNodeClearAlarmConfig",g.default).directive("tbActionNodeLogConfig",f.default).directive("tbActionNodeRpcReplyConfig",v.default).directive("tbActionNodeRpcRequestConfig",h.default).directive("tbActionNodeRestApiCallConfig",k.default).directive("tbActionNodeKafkaConfig",$.default).directive("tbActionNodeSnsConfig",C.default).directive("tbActionNodeSqsConfig",M.default).directive("tbActionNodeRabbitMqConfig",S.default).directive("tbActionNodeMqttConfig",F.default).directive("tbActionNodeSendEmailConfig",P.default).directive("tbActionNodeMsgDelayConfig",V.default).directive("tbActionNodeMsgCountConfig",j.default).directive("tbActionNodeAssignToCustomerConfig",D.default).directive("tbActionNodeUnAssignToCustomerConfig",R.default).directive("tbActionNodeDeleteRelationConfig",U.default).directive("tbActionNodeCreateRelationConfig",H.default).directive("tbActionNodeCustomTableConfig",G.default).directive("tbActionNodeGpsGeofencingConfig",Q.default).directive("tbActionNodePubSubConfig",J.default).directive("tbActionNodeCheckPointConfig",X.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ackValues=["all","-1","0","1"],n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue,n.configuration.hasOwnProperty("kafkaHeadersCharset")||(n.configuration.kafkaHeadersCharset="UTF-8")},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(15),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"string",t.instant("tb.rulenode.to-string")+"","ToString",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(16),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$mdExpansionPanel=t,a.ruleNodeTypes=n,a.credentialsTypeChanged=function(){var e=a.configuration.credentials.type;a.configuration.credentials={},a.configuration.credentials.type=e,a.updateValidity()},a.certFileAdded=function(e,t){var n=new FileReader;n.onload=function(n){a.$apply(function(){if(n.target.result){l.$setDirty();var i=n.target.result;i&&i.length>0&&("caCert"==t&&(a.configuration.credentials.caCertFileName=e.name,a.configuration.credentials.caCert=i),"privateKey"==t&&(a.configuration.credentials.privateKeyFileName=e.name,a.configuration.credentials.privateKey=i),"Cert"==t&&(a.configuration.credentials.certFileName=e.name,a.configuration.credentials.cert=i)),a.updateValidity()}})},n.readAsText(e.file)},a.clearCertFile=function(e){l.$setDirty(),"caCert"==e&&(a.configuration.credentials.caCertFileName=null,a.configuration.credentials.caCert=null),"privateKey"==e&&(a.configuration.credentials.privateKeyFileName=null,a.configuration.credentials.privateKey=null),"Cert"==e&&(a.configuration.credentials.certFileName=null,a.configuration.credentials.cert=null),a.updateValidity()},a.updateValidity=function(){var e=!0,t=a.configuration.credentials;t.type==n.mqttCredentialTypes["cert.PEM"].value&&(t.caCert&&t.cert&&t.privateKey||(e=!1)),l.$setValidity("Certs",e)},a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:a}}i.$inject=["$compile","$mdExpansionPanel","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(2);var r=n(17),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(18),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){ +var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(19),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.serviceAccountFileAdded=function(e){var t=new FileReader;t.onload=function(t){n.$apply(function(){if(t.target.result){r.$setDirty();var a=t.target.result;a&&a.length>0&&(n.configuration.serviceAccountKeyFileName=e.name,n.configuration.serviceAccountKey=a),n.updateValidity()}})},t.readAsText(e.file)},n.clearServiceAccountFile=function(){r.$setDirty(),n.configuration.serviceAccountKeyFileName=null,n.configuration.serviceAccountKey=null,n.updateValidity()},n.updateValidity=function(){var e=!0,t=n.configuration;t.serviceAccountKeyFileName&&t.serviceAccountKey||(e=!1),r.$setValidity("SAKey",e)},n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(20),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(21),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(22),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(23),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(24),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(25),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.smtpProtocols=["smtp","smtps"],t.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(26),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(27),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(28),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(29),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(30),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(31),o=a(r)},function(e,t){"use strict";function n(e){var t=function(t,n,a,i){n.html("
"),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}n.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(32),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(33),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.entityDetailsList=[];for(var s in t.entityDetails){var d=s;n.entityDetailsList.push(d)}r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(34),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s);var d=186;a.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,d],a.ruleNodeTypes=n,a.aggPeriodTimeUnits={},a.aggPeriodTimeUnits.MINUTES=n.timeUnit.MINUTES,a.aggPeriodTimeUnits.HOURS=n.timeUnit.HOURS,a.aggPeriodTimeUnits.DAYS=n.timeUnit.DAYS,a.aggPeriodTimeUnits.MILLISECONDS=n.timeUnit.MILLISECONDS,a.aggPeriodTimeUnits.SECONDS=n.timeUnit.SECONDS,a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{},link:a}}i.$inject=["$compile","$mdConstant","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(35),o=a(r);n(3)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(86),r=a(i),o=n(87),l=a(o),s=n(82),d=a(s),u=n(88),c=a(u),m=n(81),g=a(m),p=n(89),f=a(p),b=n(84),v=a(b),y=n(83),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.enrichment",[]).directive("tbEnrichmentNodeOriginatorAttributesConfig",r.default).directive("tbEnrichmentNodeOriginatorFieldsConfig",l.default).directive("tbEnrichmentNodeDeviceAttributesConfig",d.default).directive("tbEnrichmentNodeRelatedAttributesConfig",c.default).directive("tbEnrichmentNodeCustomerAttributesConfig",g.default).directive("tbEnrichmentNodeTenantAttributesConfig",f.default).directive("tbEnrichmentNodeGetTelemetryFromDatabase",v.default).directive("tbEnrichmentNodeEntityDetailsConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(36),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(37),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(38),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(39),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.alarmStatusList=[];for(var s in t.alarmStatus)n.alarmStatusList.push(t.alarmStatus[s]);r.$render=function(){n.configuration=r.$viewValue},n.getAlarmStatusList=function(){return n.alarmStatusList.filter(function(e){return n.configuration.alarmStatusList.indexOf(e)===-1})},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(40),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(41),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(42),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(43),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(97),r=a(i),o=n(95),l=a(o),s=n(98),d=a(s),u=n(92),c=a(u),m=n(96),g=a(m),p=n(91),f=a(p),b=n(93),v=a(b),y=n(90),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.filter",[]).directive("tbFilterNodeScriptConfig",r.default).directive("tbFilterNodeMessageTypeConfig",l.default).directive("tbFilterNodeSwitchConfig",d.default).directive("tbFilterNodeCheckRelationConfig",c.default).directive("tbFilterNodeOriginatorTypeConfig",g.default).directive("tbFilterNodeCheckMessageConfig",f.default).directive("tbFilterNodeGpsGeofencingConfig",v.default).directive("tbFilterNodeCheckAlarmStatusConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){function s(){if(l.$viewValue){for(var e=[],t=0;t-1&&t.kvList.splice(e,1)}function l(){t.kvList||(t.kvList=[]),t.kvList.push({key:"",value:""})}function s(){var e={};t.kvList.forEach(function(t){t.key&&(e[t.key]=t.value)}),i.$setViewValue(e),d()}function d(){var e=!0;t.required&&!t.kvList.length&&(e=!1),i.$setValidity("kvMap",e)}var u=o.default;n.html(u),t.ngModelCtrl=i,t.removeKeyVal=r,t.addKeyVal=l,t.kvList=[],t.$watch("query",function(e,n){angular.equals(e,n)||i.$setViewValue(t.query)}),i.$render=function(){if(i.$viewValue){var e=i.$viewValue;t.kvList.length=0;for(var n in e)t.kvList.push({key:n,value:e[n]})}t.$watch("kvList",function(e,t){angular.equals(e,t)||s()},!0),d()},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",disabled:"=ngDisabled",requiredText:"=",keyText:"=",keyRequiredText:"=",valText:"=",valRequiredText:"="},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(48),o=a(r);n(5)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(49),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(50),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(101),r=a(i),o=n(103),l=a(o),s=n(104),d=a(s);t.default=angular.module("thingsboard.ruleChain.config.transform",[]).directive("tbTransformationNodeChangeOriginatorConfig",r.default).directive("tbTransformationNodeScriptConfig",l.default).directive("tbTransformationNodeToEmailConfig",d.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"update",t.instant("tb.rulenode.transformer")+"","Transform",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(51),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(52),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(108),r=a(i),o=n(94),l=a(o),s=n(85),d=a(s),u=n(102),c=a(u),m=n(62),g=a(m),p=n(80),f=a(p),b=n(100),v=a(b),y=n(79),h=a(y),q=n(99),k=a(q),x=n(107),$=a(x);t.default=angular.module("thingsboard.ruleChain.config",[r.default,l.default,d.default,c.default,g.default]).directive("tbNodeEmptyConfig",f.default).directive("tbRelationsQueryConfig",v.default).directive("tbDeviceRelationsQueryConfig",h.default).directive("tbKvMapConfig",k.default).config($.default).name},function(e,t){"use strict";function n(e){var t={tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647'.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-statuses-filter":"Alarm statuses filter","alarm-statuses-required":"Alarm statuses is required",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required", +"endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata",header:"Header","header-required":"Header is required",value:"Value","value-required":"Value is required","topic-pattern":"Topic pattern","topic-pattern-required":"Topic pattern is required","mqtt-topic-pattern-hint":"MQTT topic pattern, use ${metaKeyName} to substitute variables from metadata","bootstrap-servers":"Bootstrap servers","bootstrap-servers-required":"Bootstrap servers value is required","other-properties":"Other properties",key:"Key","key-required":"Key is required",retries:"Automatically retry times if fails","min-retries-message":"Only 0 minimum retries is allowed.","batch-size-bytes":"Produces batch size in bytes","min-batch-size-bytes-message":"Only 0 minimum batch size is allowed.","linger-ms":"Time to buffer locally (ms)","min-linger-ms-message":"Only 0 ms minimum value is allowed.","buffer-memory-bytes":"Client buffer max size in bytes","min-buffer-memory-message":"Only 0 minimum buffer size is allowed.",acks:"Number of acknowledgments","key-serializer":"Key serializer","key-serializer-required":"Key serializer is required","value-serializer":"Value serializer","value-serializer-required":"Value serializer is required","topic-arn-pattern":"Topic ARN pattern","topic-arn-pattern-required":"Topic ARN pattern is required","topic-arn-pattern-hint":"Topic ARN pattern, use ${metaKeyName} to substitute variables from metadata","aws-access-key-id":"AWS Access Key ID","aws-access-key-id-required":"AWS Access Key ID is required","aws-secret-access-key":"AWS Secret Access Key","aws-secret-access-key-required":"AWS Secret Access Key is required","aws-region":"AWS Region","aws-region-required":"AWS Region is required","exchange-name-pattern":"Exchange name pattern","routing-key-pattern":"Routing key pattern","message-properties":"Message properties",host:"Host","host-required":"Host is required",port:"Port","port-required":"Port is required","port-range":"Port should be in a range from 1 to 65535.","virtual-host":"Virtual host",username:"Username",password:"Password","automatic-recovery":"Automatic recovery","connection-timeout-ms":"Connection timeout (ms)","min-connection-timeout-ms-message":"Only 0 ms minimum value is allowed.","handshake-timeout-ms":"Handshake timeout (ms)","min-handshake-timeout-ms-message":"Only 0 ms minimum value is allowed.","client-properties":"Client properties","queue-url-pattern":"Queue URL pattern","queue-url-pattern-required":"Queue URL pattern is required","queue-url-pattern-hint":"Queue URL pattern, use ${metaKeyName} to substitute variables from metadata","delay-seconds":"Delay (seconds)","min-delay-seconds-message":"Only 0 seconds minimum value is allowed.","max-delay-seconds-message":"Only 900 seconds maximum value is allowed.",name:"Name","name-required":"Name is required","queue-type":"Queue type","sqs-queue-standard":"Standard","sqs-queue-fifo":"FIFO","gcp-project-id":"GCP project ID","gcp-project-id-required":"GCP project ID is required","gcp-service-account-key":"GCP service account key file","gcp-service-account-key-required":"GCP service account key file is required","pubsub-topic-name":"Topic name","pubsub-topic-name-required":"Topic name is required","message-attributes":"Message attributes","message-attributes-hint":"Use ${metaKeyName} in name/value fields to substitute variables from metadata","connect-timeout":"Connection timeout (sec)","connect-timeout-required":"Connection timeout is required.","connect-timeout-range":"Connection timeout should be in a range from 1 to 200.","client-id":"Client ID","clean-session":"Clean session","enable-ssl":"Enable SSL",credentials:"Credentials","credentials-type":"Credentials type","credentials-type-required":"Credentials type is required.","credentials-anonymous":"Anonymous","credentials-basic":"Basic","credentials-pem":"PEM","username-required":"Username is required.","password-required":"Password is required.","ca-cert":"CA certificate file *","private-key":"Private key file *",cert:"Certificate file *","no-file":"No file selected.","drop-file":"Drop a file or click to select a file to upload.","private-key-password":"Private key password","use-system-smtp-settings":"Use system SMTP settings","use-metadata-interval-patterns":"Use metadata interval patterns","use-metadata-interval-patterns-hint":"If selected, rule node use start and end interval patterns from message metadata assuming that intervals are in the milliseconds.","use-message-alarm-data":"Use message alarm data","check-all-keys":"Check that all selected keys are present","check-all-keys-hint":"If selected, checks that all specified keys are present in the message data and metadata.","check-relation-to-specific-entity":"Check relation to specific entity","check-relation-hint":"Checks existence of relation to specific entity or to any entity based on direction and relation type.","delete-relation-to-specific-entity":"Delete relation to specific entity","delete-relation-hint":"Deletes relation from the originator of the incoming message to the specified entity or list of entities based on direction and type.","remove-current-relations":"Remove current relations","remove-current-relations-hint":"Removes current relations from the originator of the incoming message based on direction and type.","change-originator-to-related-entity":"Change originator to related entity","change-originator-to-related-entity-hint":"Used to process submitted message as a message from another entity.","start-interval-pattern":"Start interval pattern","end-interval-pattern":"End interval pattern","start-interval-pattern-required":"Start interval pattern is required","end-interval-pattern-required":"End interval pattern is required","start-interval-pattern-hint":"Start interval pattern, use ${metaKeyName} to substitute variables from metadata","end-interval-pattern-hint":"End interval pattern, use ${metaKeyName} to substitute variables from metadata","smtp-protocol":"Protocol","smtp-host":"SMTP host","smtp-host-required":"SMTP host is required.","smtp-port":"SMTP port","smtp-port-required":"You must supply a smtp port.","smtp-port-range":"SMTP port should be in a range from 1 to 65535.","timeout-msec":"Timeout ms","min-timeout-msec-message":"Only 0 ms minimum value is allowed.","enter-username":"Enter username","enter-password":"Enter password","enable-tls":"Enable TLS","tls-version":"TLS version","min-period-0-seconds-message":"Only 0 second minimum period is allowed.","max-pending-messages":"Maximum pending messages","max-pending-messages-required":"Maximum pending messages is required.","max-pending-messages-range":"Maximum pending messages should be in a range from 1 to 100000.","originator-types-filter":"Originator types filter","interval-seconds":"Interval in seconds","interval-seconds-required":"Interval is required.","min-interval-seconds-message":"Only 1 second minimum interval is allowed.","output-timeseries-key-prefix":"Output timeseries key prefix","output-timeseries-key-prefix-required":"Output timeseries key prefix required.","separator-hint":'You should press "enter" to complete field input.',"entity-details":"Select entity details:","entity-details-title":"Title","entity-details-country":"Country","entity-details-state":"State","entity-details-zip":"Zip","entity-details-address":"Address","entity-details-address2":"Address2","entity-details-additional_info":"Additional Info","entity-details-phone":"Phone","entity-details-email":"Email","add-to-metadata":"Add selected details to message metadata","add-to-metadata-hint":"If selected, adds the selected details keys to the message metadata instead of message data.","entity-details-list-empty":"No entity details selected.","no-entity-details-matching":"No entity details matching were found.","custom-table-name":"Custom table name","custom-table-name-required":"Table Name is required","custom-table-hint":"You should enter the table name without prefix 'cs_tb_'.","message-field":"Message field","message-field-required":"Message field is required.","table-col":"Table column","table-col-required":"Table column is required.","latitude-key-name":"Latitude key name","longitude-key-name":"Longitude key name","latitude-key-name-required":"Latitude key name is required.","longitude-key-name-required":"Longitude key name is required.","fetch-perimeter-info-from-message-metadata":"Fetch perimeter information from message metadata","perimeter-circle":"Circle","perimeter-polygon":"Polygon","perimeter-type":"Perimeter type","circle-center-latitude":"Center latitude","circle-center-latitude-required":"Center latitude is required.","circle-center-longitude":"Center longitude","circle-center-longitude-required":"Center longitude is required.","range-unit-meter":"Meter","range-unit-kilometer":"Kilometer","range-unit-foot":"Foot","range-unit-mile":"Mile","range-unit-nautical-mile":"Nautical mile","range-units":"Range units",range:"Range","range-required":"Range is required.","polygon-definition":"Polygon definition","polygon-definition-required":"Polygon definition is required.","polygon-definition-hint":"Please, use the following format for manual definition of polygon: [[lat1,lon1],[lat2,lon2], ... ,[latN,lonN]].","min-inside-duration":"Minimal inside duration","min-inside-duration-value-required":"Minimal inside duration is required","min-inside-duration-time-unit":"Minimal inside duration time unit","min-outside-duration":"Minimal outside duration","min-outside-duration-value-required":"Minimal outside duration is required","min-outside-duration-time-unit":"Minimal outside duration time unit","tell-failure-if-absent":"Tell Failure","tell-failure-if-absent-hint":'If at least one selected key doesn\'t exist the outbound message will report "Failure".',"get-latest-value-with-ts":"Fetch Latest telemetry with Timestamp","get-latest-value-with-ts-hint":'If selected, latest telemetry values will be added to the outbound message metadata with timestamp, e.g: "temp": "{\\"ts\\":1574329385897,\\"value\\":42}"',"use-redis-queue":"Use redis queue for message persistence","trim-redis-queue":"Trim redis queue","redis-queue-max-size":"Redis queue max size","add-metadata-key-values-as-kafka-headers":"Add Message metadata key-value pairs to Kafka record headers","add-metadata-key-values-as-kafka-headers-hint":"If selected, key-value pairs from message metadata will be added to the outgoing records headers as byte arrays with predefined charset encoding.","charset-encoding":"Charset encoding","charset-encoding-required":"Charset encoding is required.","charset-us-ascii":"US-ASCII","charset-iso-8859-1":"ISO-8859-1","charset-utf-8":"UTF-8","charset-utf-16be":"UTF-16BE","charset-utf-16le":"UTF-16LE","charset-utf-16":"UTF-16","select-queue-hint":"The queue name can be selected from a drop-down list or add a custom name."},"key-val":{key:"Key",value:"Value","remove-entry":"Remove entry","add-entry":"Add entry"}}};e.translations("en_US",t)}Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){(0,o.default)(e)}i.$inject=["$translateProvider"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(106),o=a(r)},function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=angular.module("thingsboard.ruleChain.config.types",[]).constant("ruleNodeTypes",{originatorSource:{CUSTOMER:{name:"tb.rulenode.originator-customer",value:"CUSTOMER"},TENANT:{name:"tb.rulenode.originator-tenant",value:"TENANT"},RELATED:{name:"tb.rulenode.originator-related",value:"RELATED"},ALARM_ORIGINATOR:{name:"tb.rulenode.originator-alarm-originator",value:"ALARM_ORIGINATOR"}},fetchModeType:["FIRST","LAST","ALL"],samplingOrder:["ASC","DESC"],httpRequestType:["GET","POST","PUT","DELETE"],entityDetails:{TITLE:{name:"tb.rulenode.entity-details-title",value:"TITLE"},COUNTRY:{name:"tb.rulenode.entity-details-country",value:"COUNTRY"},STATE:{name:"tb.rulenode.entity-details-state",value:"STATE"},ZIP:{name:"tb.rulenode.entity-details-zip",value:"ZIP"},ADDRESS:{name:"tb.rulenode.entity-details-address",value:"ADDRESS"},ADDRESS2:{name:"tb.rulenode.entity-details-address2",value:"ADDRESS2"},PHONE:{name:"tb.rulenode.entity-details-phone",value:"PHONE"},EMAIL:{name:"tb.rulenode.entity-details-email",value:"EMAIL"},ADDITIONAL_INFO:{name:"tb.rulenode.entity-details-additional_info",value:"ADDITIONAL_INFO"}},sqsQueueType:{STANDARD:{name:"tb.rulenode.sqs-queue-standard",value:"STANDARD"},FIFO:{name:"tb.rulenode.sqs-queue-fifo",value:"FIFO"}},perimeterType:{CIRCLE:{name:"tb.rulenode.perimeter-circle",value:"CIRCLE"},POLYGON:{name:"tb.rulenode.perimeter-polygon",value:"POLYGON"}},timeUnit:{MILLISECONDS:{value:"MILLISECONDS",name:"tb.rulenode.time-unit-milliseconds"},SECONDS:{value:"SECONDS",name:"tb.rulenode.time-unit-seconds"},MINUTES:{value:"MINUTES",name:"tb.rulenode.time-unit-minutes"},HOURS:{value:"HOURS",name:"tb.rulenode.time-unit-hours"},DAYS:{value:"DAYS",name:"tb.rulenode.time-unit-days"}},rangeUnit:{METER:{value:"METER",name:"tb.rulenode.range-unit-meter"},KILOMETER:{value:"KILOMETER",name:"tb.rulenode.range-unit-kilometer"},FOOT:{value:"FOOT",name:"tb.rulenode.range-unit-foot"},MILE:{value:"MILE",name:"tb.rulenode.range-unit-mile"},NAUTICAL_MILE:{value:"NAUTICAL_MILE",name:"tb.rulenode.range-unit-nautical-mile"}},mqttCredentialTypes:{anonymous:{value:"anonymous",name:"tb.rulenode.credentials-anonymous"},basic:{value:"basic",name:"tb.rulenode.credentials-basic"},"cert.PEM":{value:"cert.PEM",name:"tb.rulenode.credentials-pem"}},toBytesStandartCharsetTypes:{"US-ASCII":{value:"US-ASCII",name:"tb.rulenode.charset-us-ascii"},"ISO-8859-1":{value:"ISO-8859-1",name:"tb.rulenode.charset-iso-8859-1"},"UTF-8":{value:"UTF-8",name:"tb.rulenode.charset-utf-8"},"UTF-16BE":{value:"UTF-16BE",name:"tb.rulenode.charset-utf-16be"},"UTF-16LE":{value:"UTF-16LE",name:"tb.rulenode.charset-utf-16le"},"UTF-16":{value:"UTF-16",name:"tb.rulenode.charset-utf-16"}}}).name}])); //# sourceMappingURL=rulenode-core-config.js.map \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index 67c5b1cc18..61b83d647d 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -25,12 +25,15 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -38,20 +41,35 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.alarm.AlarmService; import javax.script.ScriptException; import java.io.IOException; import java.util.concurrent.Callable; +import java.util.function.Consumer; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; -import static org.mockito.Mockito.*; -import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.*; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_CLEARED_ALARM; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_EXISTING_ALARM; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_NEW_ALARM; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.CRITICAL; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.WARNING; -import static org.thingsboard.server.common.data.alarm.AlarmStatus.*; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.ACTIVE_UNACK; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_ACK; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_UNACK; @RunWith(MockitoJUnitRunner.class) public class TbAlarmNodeTest { @@ -66,6 +84,11 @@ public class TbAlarmNodeTest { @Mock private ScriptEngine detailsJs; + @Captor + private ArgumentCaptor successCaptor; + @Captor + private ArgumentCaptor> failureCaptor; + private RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); private RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); @@ -99,15 +122,16 @@ public class TbAlarmNodeTest { public void newAlarmCanBeCreated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFuture(null)); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); - doAnswer((Answer) invocationOnMock -> (Alarm) (invocationOnMock.getArguments())[0]).when(alarmService).createOrUpdateAlarm(any(Alarm.class)); node.onMsg(ctx, msg); + verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); + successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -141,7 +165,7 @@ public class TbAlarmNodeTest { public void buildDetailsThrowsException() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFailedFuture(new NotImplementedException("message"))); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -164,7 +188,7 @@ public class TbAlarmNodeTest { public void ifAlarmClearedCreateNew() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); Alarm clearedAlarm = Alarm.builder().status(CLEARED_ACK).build(); @@ -175,6 +199,8 @@ public class TbAlarmNodeTest { node.onMsg(ctx, msg); + verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); + successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -209,7 +235,7 @@ public class TbAlarmNodeTest { public void alarmCanBeUpdated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); @@ -256,7 +282,7 @@ public class TbAlarmNodeTest { public void alarmCanBeCleared() throws ScriptException, IOException { initWithClearAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java index 27feed93ba..b0f64fb980 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java @@ -19,7 +19,6 @@ import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; @@ -28,17 +27,24 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class TbJsFilterNodeTest { @@ -58,7 +64,7 @@ public class TbJsFilterNodeTest { @Test public void falseEvaluationDoNotSendMsg() throws TbNodeException, ScriptException { initWithScript(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", null, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(false)); @@ -71,7 +77,7 @@ public class TbJsFilterNodeTest { public void exceptionInJsThrowsException() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFailedFuture(new ScriptException("error"))); @@ -84,7 +90,7 @@ public class TbJsFilterNodeTest { public void metadataConditionCanBeTrue() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(true)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index 1e752b55b8..9286a06da6 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -28,10 +28,14 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -40,7 +44,9 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class TbJsSwitchNodeTest { @@ -65,7 +71,7 @@ public class TbJsSwitchNodeTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeSwitch(msg)).thenReturn(Sets.newHashSet("one", "three")); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java index ace02bacd7..70f58c5989 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java @@ -30,13 +30,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) @@ -62,7 +62,7 @@ public class TbMsgToEmailNodeTest { metaData.putValue("name", "temp"); metaData.putValue("passed", "5"); metaData.putValue("count", "100"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); emailNode.onMsg(ctx, msg); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index 0af1cef8ae..a0a61ae30f 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -31,9 +31,19 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.*; -import org.thingsboard.server.common.data.kv.*; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -102,7 +112,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -127,7 +137,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -152,7 +162,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(null)); @@ -166,7 +176,7 @@ public class TbGetCustomerAttributeNodeTest { @Test public void customerAttributeAddedInMetadata() { CustomerId customerId = new CustomerId(UUIDs.timeBased()); - msg = new TbMsg(UUIDs.timeBased(), "CUSTOMER", customerId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "CUSTOMER", customerId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); entityAttributeFetched(customerId); } @@ -177,7 +187,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -192,7 +202,7 @@ public class TbGetCustomerAttributeNodeTest { Asset asset = new Asset(); asset.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -207,7 +217,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -234,7 +244,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -246,7 +256,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(timeseries)); node.onMsg(ctx, msg); - verify(ctx).tellNext(msg, SUCCESS); + verify(ctx).tellSuccess(msg); assertEquals(msg.getMetaData().getValue("tempo"), "highest"); } @@ -258,7 +268,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(attributes)); node.onMsg(ctx, msg); - verify(ctx).tellNext(msg, SUCCESS); + verify(ctx).tellSuccess(msg); assertEquals(msg.getMetaData().getValue("tempo"), "high"); } } \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java index 4c1edce8f6..fa845c4a18 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; @@ -91,7 +92,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(),eq( assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -119,7 +120,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -146,7 +147,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(null)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index aa9ad8b76f..a00e97cf1c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -27,10 +27,14 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -38,7 +42,10 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @RunWith(MockitoJUnitRunner.class) @@ -62,15 +69,15 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); - TbMsg transformedMsg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{new}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON,rawJson, ruleChainId, ruleNodeId); + TbMsg transformedMsg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{new}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFuture(transformedMsg)); node.onMsg(ctx, msg); verify(ctx).getDbCallbackExecutor(); ArgumentCaptor captor = ArgumentCaptor.forClass(TbMsg.class); - verify(ctx).tellNext(captor.capture(), eq(SUCCESS)); + verify(ctx).tellSuccess(captor.capture()); TbMsg actualMsg = captor.getValue(); assertEquals(transformedMsg, actualMsg); } @@ -84,7 +91,7 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFailedFuture(new IllegalStateException("error"))); diff --git a/tools/pom.xml b/tools/pom.xml index 64027c478e..1960d330d0 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -54,7 +54,7 @@ org.apache.cassandra cassandra-all - 3.11.4 + 3.11.6 com.datastax.cassandra diff --git a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java index 6dbca324e1..ec596db10d 100644 --- a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java +++ b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.queue.kafka"}) public class ThingsboardCoapTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 9b3572c7d1..c3dcb76508 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -41,24 +41,143 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java index 39dc0677e8..f44af2105e 100644 --- a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java +++ b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java @@ -24,7 +24,7 @@ import java.util.Arrays; @SpringBootApplication @EnableAsync -@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.queue.kafka"}) public class ThingsboardHttpTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 4097bce831..baac6e977f 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -42,24 +42,143 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java index ce95ee6f1f..cd7225ad1f 100644 --- a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java +++ b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.queue"}) public class ThingsboardMqttTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index d13a3d071e..e920891d66 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -17,10 +17,20 @@ spring.main.web-environment: false spring.main.web-application-type: none -# Clustering properties -cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:false}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # MQTT server parameters transport: @@ -62,24 +72,143 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/ui/package-lock.json b/ui/package-lock.json index dccf637726..512bac811c 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1095,12 +1095,12 @@ "@flowjs/ng-flow": { "version": "2.7.8", "resolved": "https://registry.npmjs.org/@flowjs/ng-flow/-/ng-flow-2.7.8.tgz", - "integrity": "sha512-zO6jNvz41oMOJj9+1N+vLT0ytitbCtuGABJQRzQDOPXyRMmlSXfJ7om5oYOztyUFrr4jDpE4QFPt+r2/RFceCg==" + "integrity": "sha1-HZ+dH4Ks2lNgMowxW6z9YNv9mBk=" }, "@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", - "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==", + "integrity": "sha1-UkryQNGjYFJ7cwR17PoTRKpUDd4=", "dev": true, "requires": { "call-me-maybe": "^1.0.1", @@ -1424,7 +1424,7 @@ "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + "integrity": "sha1-+PLIh60Qv2f2NPAFtph/7TF5qsg=" }, "accepts": { "version": "1.3.7", @@ -1534,7 +1534,7 @@ "angular-carousel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/angular-carousel/-/angular-carousel-1.1.0.tgz", - "integrity": "sha512-UiLMgT7Ueqk4xpliF1gWt4dYKXezdJA1jyZPNsUWkOGO/dwLuKi284h3BgWl4CnaH7kEBw8L2gsBOyqbYaumNQ==" + "integrity": "sha1-PmlA5ovRio85L8Qx2XGSrDSIMdE=" }, "angular-cookies": { "version": "1.5.8", @@ -1555,7 +1555,7 @@ } }, "angular-fullscreen": { - "version": "git://github.com/fabiobiondi/angular-fullscreen.git#119b7fbac911d154fd56ace38ebe3432475e8a20", + "version": "git://github.com/fabiobiondi/angular-fullscreen.git#8217174565761d3566807bc60a73b5ca015b8cb6", "from": "git://github.com/fabiobiondi/angular-fullscreen.git#master" }, "angular-gridster": { @@ -1629,7 +1629,7 @@ "angular-translate": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate/-/angular-translate-2.18.1.tgz", - "integrity": "sha512-Mw0kFBqsv5j8ItL9IhRZunIlVmIRW6iFsiTmRs9wGr2QTt8z4rehYlWyHos8qnXc/kyOYJiW50iH50CSNHGB9A==", + "integrity": "sha1-sp7Q0vm6xEB156rTKEFmxZ4VB5E=", "requires": { "angular": ">=1.2.26 <=1.7" } @@ -1637,7 +1637,7 @@ "angular-translate-handler-log": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-handler-log/-/angular-translate-handler-log-2.18.1.tgz", - "integrity": "sha512-TyKzCW4GubNazwCgLpCVXd2212CWdZOckf+aL5+gLuThPhVpOvlg18RSmz8MNPto3kwCcCw3LzShlZ6RX/MQRA==", + "integrity": "sha1-icu1mCeALYb4EVJ1+/iNbYiWsNQ=", "requires": { "angular-translate": "~2.18.1" } @@ -1645,7 +1645,7 @@ "angular-translate-interpolation-messageformat": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-interpolation-messageformat/-/angular-translate-interpolation-messageformat-2.18.1.tgz", - "integrity": "sha512-SlmyxLB/UUy7FWoGx5QJHrhq8fUu/xzCR0h/ngexOtXZopQjs1vm+TrFZ69d4c/LI7C91sfP4mq4ES29o1xCxA==", + "integrity": "sha1-FsUq4MYcJA8PJBZKBSGUPPi6QI4=", "requires": { "angular-translate": "~2.18.1", "messageformat": "~1.0.2" @@ -1654,7 +1654,7 @@ "angular-translate-loader-static-files": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-loader-static-files/-/angular-translate-loader-static-files-2.18.1.tgz", - "integrity": "sha512-5MuyzAROfc493kjLjKlLGLBzXiRmZIFbcWZGutDRxW5SRXSpwrH0u0hh0ENNnUyUQbe2vUspHNPIuZqlq8qIhw==", + "integrity": "sha1-rQw8iDsYsIm9uNsCu9Nm2QP4V8w=", "requires": { "angular-translate": "~2.18.1" } @@ -1662,7 +1662,7 @@ "angular-translate-storage-cookie": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-cookie/-/angular-translate-storage-cookie-2.18.1.tgz", - "integrity": "sha512-wiMaF/0OGN/3ilaYunfsqdLNpfGZEJK0fj4zT8yjD3XPq7Q9kM88xZ4XJiWKgodZShBljGCRzqgQbKMF7d1MLw==", + "integrity": "sha1-j8vaspb6gkkOALQorxp0ahf0QVY=", "requires": { "angular-cookies": ">=1.2.26 <1.8", "angular-translate": "~2.18.1" @@ -1671,7 +1671,7 @@ "angular-translate-storage-local": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-local/-/angular-translate-storage-local-2.18.1.tgz", - "integrity": "sha512-zPxcbIJ8tdWXtWNKLtaswynKid0w5le6WPMwiLWhgKPnyzOp/y5WLBW+JEfnZnkGE24yOGhJ6jVPgRNzelLgzg==", + "integrity": "sha1-lHQP5NgBq3gpopofBeHDkFTIcwM=", "requires": { "angular-translate": "~2.18.1", "angular-translate-storage-cookie": "~2.18.1" @@ -1750,7 +1750,7 @@ "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", + "integrity": "sha1-aALmJk79GMeQobDVF/DyYnvyyUo=", "dev": true }, "are-we-there-yet": { @@ -1766,7 +1766,7 @@ "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=", "dev": true, "requires": { "sprintf-js": "~1.0.2" @@ -1781,7 +1781,7 @@ "arr-flatten": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", - "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "integrity": "sha1-NgSLv/TntH4TZkQxbJlmnqWukfE=", "dev": true }, "arr-union": { @@ -1893,7 +1893,7 @@ }, "util": { "version": "0.10.3", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1958,7 +1958,7 @@ "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", - "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "integrity": "sha1-bZUX654DDSQ2ZmZR6GvZ9vE1M8k=", "dev": true }, "attr-accept": { @@ -2174,7 +2174,7 @@ "base": { "version": "0.11.2", "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", - "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "integrity": "sha1-e95c7RRbbVUakNuH+DxVi060io8=", "dev": true, "requires": { "cache-base": "^1.0.1", @@ -2198,7 +2198,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2207,7 +2207,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2216,7 +2216,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -2362,7 +2362,7 @@ "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=", "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2586,7 +2586,7 @@ "cache-base": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", - "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "integrity": "sha1-Cn9GQWgxyLZi7jb+TnxZ129marI=", "dev": true, "requires": { "collection-visit": "^1.0.0", @@ -2617,7 +2617,7 @@ "dependencies": { "callsites": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", + "resolved": "http://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", "integrity": "sha1-BuuE8A7qQT2oav/vrL/7Ngk7PFA=", "dev": true } @@ -2794,7 +2794,7 @@ "class-utils": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", - "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "integrity": "sha1-+TNprouafOAv1B+q0MqDAzGQxGM=", "dev": true, "requires": { "arr-union": "^3.1.0", @@ -3121,7 +3121,7 @@ "concat-stream": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "integrity": "sha1-kEvfGUzTEi/Gdcd/xKw9T/D9GjQ=", "dev": true, "requires": { "buffer-from": "^1.0.0", @@ -3181,7 +3181,7 @@ "content-type": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "integrity": "sha1-4TjMdeBAxyexlm/l5fjJruJW/js=", "dev": true }, "convert-source-map": { @@ -3208,7 +3208,7 @@ "copy-concurrently": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz", - "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==", + "integrity": "sha1-kilzmMrjSTf8r9bsgTnBgFHwteA=", "dev": true, "requires": { "aproba": "^1.1.1", @@ -3427,7 +3427,7 @@ "create-react-class": { "version": "15.6.3", "resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.6.3.tgz", - "integrity": "sha512-M+/3Q6E6DLO6Yx3OwrWjwHBnvfXXYA7W+dFjt/ZDBemHO1DDZhsalX/NUtnTYclN6GfnBDRh4qRHjcDHmlJBJg==", + "integrity": "sha1-LXMjf7P5cK5uvgEanmb0bbyoADY=", "requires": { "fbjs": "^0.8.9", "loose-envify": "^1.3.1", @@ -3555,7 +3555,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", "dev": true, "requires": { "ms": "2.0.0" @@ -3633,7 +3633,7 @@ "define-property": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", - "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "integrity": "sha1-1Flono1lS6d+AqgX+HENcCyxbp0=", "dev": true, "requires": { "is-descriptor": "^1.0.2", @@ -3643,7 +3643,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3652,7 +3652,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3661,7 +3661,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -3724,7 +3724,7 @@ "delegate": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", - "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" + "integrity": "sha1-tmtxwxWFIuirV0T3INjKDCr1kWY=" }, "delegates": { "version": "1.0.0", @@ -3880,7 +3880,7 @@ "domain-browser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", - "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", + "integrity": "sha1-PTH1AZGmdJ3RN1p/Ui6CPULlTto=", "dev": true }, "domelementtype": { @@ -4035,7 +4035,7 @@ "errno": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", - "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", + "integrity": "sha1-RoTXF3mtOa8Xfj8AeZb3xnyFJhg=", "dev": true, "requires": { "prr": "~1.0.1" @@ -4554,7 +4554,7 @@ "esrecurse": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", - "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "integrity": "sha1-AHo7n9vCs7uH5IeeoZyS/b05Qs8=", "dev": true, "requires": { "estraverse": "^4.1.0" @@ -4769,7 +4769,7 @@ "external-editor": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz", - "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==", + "integrity": "sha1-BFURz9jRM/OEZnPRBHwVTiFK09U=", "requires": { "chardet": "^0.4.0", "iconv-lite": "^0.4.17", @@ -5197,7 +5197,7 @@ "fs-readdir-recursive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz", - "integrity": "sha512-GNanXlVr2pf02+sPN40XN8HG+ePaNcvM0q5mZBd668Obwb0yD5GiUbZOFgwn8kGMY6I3mdyDJzieUy3PTYyTRA==", + "integrity": "sha1-4y/AMKLM7kSmtTcTCNpUvgs5fSc=", "dev": true }, "fs-write-stream-atomic": { @@ -5238,8 +5238,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -5260,14 +5259,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5282,20 +5279,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -5412,8 +5406,7 @@ "inherits": { "version": "2.0.4", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -5425,7 +5418,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5440,7 +5432,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5448,14 +5439,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.9.0", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5474,7 +5463,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -5564,8 +5552,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -5577,7 +5564,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -5663,8 +5649,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -5700,7 +5685,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5720,7 +5704,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5764,14 +5747,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, @@ -5790,7 +5771,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "integrity": "sha1-pWiZ0+o8m6uHS7l3O3xe3pL0iV0=", "dev": true }, "functional-red-black-tree": { @@ -6738,7 +6719,7 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "integrity": "sha1-7uJfVtscnsYIXgwid4CD9Zar+Sc=" }, "inline-style-prefixer": { "version": "2.0.5", @@ -6788,7 +6769,7 @@ "invariant": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", - "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "integrity": "sha1-YQ88ksk1nOHbYW5TgAjSP/NRWOY=", "dev": true, "requires": { "loose-envify": "^1.0.0" @@ -6890,7 +6871,7 @@ "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "integrity": "sha1-76ouqdqg16suoTqXsritUf776L4=", "dev": true }, "is-callable": { @@ -6934,7 +6915,7 @@ "is-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", - "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "integrity": "sha1-Nm2CQN3kh8pRgjsaufB6EKeCUco=", "dev": true, "requires": { "is-accessor-descriptor": "^0.1.6", @@ -6945,7 +6926,7 @@ "kind-of": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", - "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "integrity": "sha1-cpyR4thXt6QZofmqZWhcTDP1hF0=", "dev": true } } @@ -7053,7 +7034,7 @@ "is-plain-object": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", - "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "integrity": "sha1-LBY7P6+xtgbZ0Xko8FwqHDjgdnc=", "dev": true, "requires": { "isobject": "^3.0.1" @@ -7120,7 +7101,7 @@ "is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "integrity": "sha1-0YUOuXkezRjmGCzhKjDzlmNLsZ0=", "dev": true }, "is-word-character": { @@ -7260,7 +7241,7 @@ "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "integrity": "sha1-u4Z8+zRQ5pEHwTHRxRS6s9yLyqk=", "dev": true }, "json-schema": { @@ -7277,7 +7258,7 @@ "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA=", "dev": true }, "json-stable-stringify-without-jsonify": { @@ -7517,7 +7498,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -8050,7 +8031,7 @@ "messageformat-parser": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-1.1.0.tgz", - "integrity": "sha512-Hwem6G3MsKDLS1FtBRGIs8T50P1Q00r3srS6QJePCFbad9fq0nYxwf3rnU2BreApRGhmpKMV7oZI06Sy1c9TPA==" + "integrity": "sha1-E7oiUKdrvejg/KDbs0dflcWUqQo=" }, "methods": { "version": "1.1.2", @@ -8092,7 +8073,7 @@ "mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "integrity": "sha1-Ms2eXGRVO9WNGaVor0Uqz/BJgbE=", "dev": true }, "mime-db": { @@ -8113,7 +8094,7 @@ "mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" + "integrity": "sha1-ggyGo5M0ZA6ZUWkovQP8qIBX0CI=" }, "min-document": { "version": "2.19.0", @@ -8170,7 +8151,7 @@ "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", "requires": { "brace-expansion": "^1.1.7" } @@ -8265,7 +8246,7 @@ "is-extendable": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "integrity": "sha1-p0cPnkJnM9gb2B4RVSZOOjUHyrQ=", "dev": true, "requires": { "is-plain-object": "^2.0.4" @@ -8343,7 +8324,7 @@ "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", - "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "integrity": "sha1-uHqKpPwN6P5r6IiVs4mD/yZb0Rk=", "dev": true, "requires": { "arr-diff": "^4.0.0", @@ -8495,7 +8476,7 @@ } }, "ngFlowchart": { - "version": "git://github.com/thingsboard/ngFlowchart.git#b941e4ed38c226019890b7b0802b71c2b147f0e0", + "version": "git://github.com/thingsboard/ngFlowchart.git#1343a7478961f68280d81f0ecda4e722a2068e0f", "from": "git://github.com/thingsboard/ngFlowchart.git#master" }, "ngclipboard": { @@ -8555,7 +8536,7 @@ "no-case": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/no-case/-/no-case-2.3.2.tgz", - "integrity": "sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==", + "integrity": "sha1-YLgTOWvjmz8SiKTB7V0efSi0ZKw=", "dev": true, "requires": { "lower-case": "^1.1.1" @@ -8574,7 +8555,7 @@ "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", - "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", + "integrity": "sha1-mA9vcthSEaU0fGsrwYxbhMPrR+8=", "requires": { "encoding": "^0.1.11", "is-stream": "^1.0.1" @@ -8775,7 +8756,7 @@ "npmlog": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "integrity": "sha1-CKfyqL9zRgR3mp76StXMcXq7lUs=", "dev": true, "requires": { "are-we-there-yet": "~1.1.2", @@ -9092,7 +9073,7 @@ "osenv": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", - "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "integrity": "sha1-hc36+uso6Gd/QW4odZK18/SepBA=", "requires": { "os-homedir": "^1.0.0", "os-tmpdir": "^1.0.0" @@ -9520,7 +9501,7 @@ "postcss-loader": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/postcss-loader/-/postcss-loader-3.0.0.tgz", - "integrity": "sha512-cLWoDEY5OwHcAjDnkyRQzAXfs2jrKjXpO/HQFcc5b5u/r7aa471wdmChmwfnv7x2u840iat/wi0lQ5nbRgSkUA==", + "integrity": "sha1-a5eUPkfHLYRfqeA/Jzdz1OjdbC0=", "dev": true, "requires": { "loader-utils": "^1.1.0", @@ -9823,7 +9804,7 @@ "private": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", - "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==", + "integrity": "sha1-I4Hts2ifelPWUxkAYPz4ItLzaP8=", "dev": true }, "process": { @@ -9846,7 +9827,7 @@ "promise": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", - "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "integrity": "sha1-BktyYCsY+Q8pGSuLG8QY/9Hr078=", "requires": { "asap": "~2.0.3" } @@ -9926,7 +9907,7 @@ "pumpify": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", - "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "integrity": "sha1-NlE74karJ1cLGjdKXOJ4v9dDcM4=", "dev": true, "requires": { "duplexify": "^3.6.0", @@ -10091,7 +10072,7 @@ "rc-menu": { "version": "5.1.4", "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-5.1.4.tgz", - "integrity": "sha512-ZUkUNda70GtTXcQDiO3rSDdk3sgIwDwzPUm5dVM8nRH/j84qv0BVBkIUwIBu8+s+G3G9lWLurRqh22dCqZPeOA==", + "integrity": "sha1-5d8I/ouDPoFGkTX/E7MKuPIf88Y=", "requires": { "babel-runtime": "6.x", "classnames": "2.x", @@ -10122,7 +10103,7 @@ "rc-trigger": { "version": "1.11.5", "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-1.11.5.tgz", - "integrity": "sha512-MBuUPw1nFzA4K7jQOwb7uvFaZFjXGd00EofUYiZ+l/fgKVq8wnLC0lkv36kwqM7vfKyftRo2sh7cWVpdPuNnnw==", + "integrity": "sha1-+I+fhODnn44O8cjRv4rCIItxViA=", "requires": { "babel-runtime": "6.x", "create-react-class": "15.x", @@ -10281,7 +10262,7 @@ "react-transition-group": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-1.2.1.tgz", - "integrity": "sha512-CWaL3laCmgAFdxdKbhhps+c0HRGF4c+hdM4H23+FI1QBNUyx/AMeIJGWorehPNSaKnQNOAxL7PQmqMu78CDj3Q==", + "integrity": "sha1-4R9yslf5IbITIpp3TfRmEjRsfKY=", "requires": { "chain-function": "^1.0.0", "dom-helpers": "^3.2.0", @@ -10293,7 +10274,7 @@ "reactcss": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/reactcss/-/reactcss-1.2.3.tgz", - "integrity": "sha512-KiwVUcFu1RErkI97ywr8nvx8dNOpT03rbnma0SSalTYjkrPYaEajR4a/MRt6DZ46K6arDRbWMNHF+xH7G7n/8A==", + "integrity": "sha1-wAATh15Vexzw39mjaKHD2rO1SN0=", "requires": { "lodash": "^4.0.1" } @@ -10476,7 +10457,7 @@ "regenerator-runtime": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", - "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" + "integrity": "sha1-vgWtf5v30i4Fb5cmzuUBf78Z4uk=" }, "regenerator-transform": { "version": "0.14.1", @@ -10490,7 +10471,7 @@ "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", - "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "integrity": "sha1-H07OJ+ALC2XgJHpoEOaoXYOldSw=", "dev": true, "requires": { "extend-shallow": "^3.0.2", @@ -10795,7 +10776,7 @@ "ret": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", - "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "integrity": "sha1-uKSCXVvbH8P29Twrwz+BOIaBx7w=", "dev": true }, "retry": { @@ -10899,7 +10880,7 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" }, "sass-graph": { "version": "2.2.4", @@ -11233,7 +11214,7 @@ "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", - "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "integrity": "sha1-ZJIufFZbDhQgS6GqfWlkJ40lGC0=", "dev": true, "requires": { "base": "^0.11.1", @@ -11269,7 +11250,7 @@ "snapdragon-node": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", - "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "integrity": "sha1-bBdfhv8UvbByRWPo88GwIaKGhTs=", "dev": true, "requires": { "define-property": "^1.0.0", @@ -11289,7 +11270,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11298,7 +11279,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11307,7 +11288,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -11320,7 +11301,7 @@ "snapdragon-util": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", - "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "integrity": "sha1-+VZHlIbyrNeXAGk/b3uAXkWrVuI=", "dev": true, "requires": { "kind-of": "^3.2.0" @@ -11340,7 +11321,7 @@ "sockjs": { "version": "0.3.19", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.19.tgz", - "integrity": "sha512-V48klKZl8T6MzatbLlzzRNhMepEys9Y4oGFpypBFFn1gLI/QQ9HtLLyWJNbPlwGLelOVOEijUbTTJeLLI59jLw==", + "integrity": "sha1-2Xa76ACve9IK4IWY1YI5NQiZPA0=", "dev": true, "requires": { "faye-websocket": "^0.10.0", @@ -11464,7 +11445,7 @@ "spdx-expression-parse": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", - "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "integrity": "sha1-meEZt6XaAOBUkcn6M4t5BII7QdA=", "dev": true, "requires": { "spdx-exceptions": "^2.1.0", @@ -11558,7 +11539,7 @@ "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", - "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "integrity": "sha1-fLCd2jqGWFcFxks5pkZgOGguj+I=", "dev": true, "requires": { "extend-shallow": "^3.0.0" @@ -11698,7 +11679,7 @@ "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "integrity": "sha1-q5Pyeo3BPSjKyBXEYhQ6bZASrp4=", "requires": { "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^4.0.0" @@ -11742,7 +11723,7 @@ "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", "requires": { "safe-buffer": "~5.1.0" } @@ -12711,7 +12692,7 @@ "symbol-observable": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", - "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==" + "integrity": "sha1-wiaIrtTqs83C3+rLtWFmBWCgCAQ=" }, "table": { "version": "5.4.6", @@ -12941,7 +12922,7 @@ "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "integrity": "sha1-bTQzWIl2jSGyvNoKonfO07G/rfk=", "requires": { "os-tmpdir": "~1.0.2" } @@ -12981,7 +12962,7 @@ "to-regex": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", - "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "integrity": "sha1-E8/dmzNlUvMLUfM6iuG0Knp1mc4=", "dev": true, "requires": { "define-property": "^2.0.2", @@ -13183,7 +13164,7 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=", "dev": true } } @@ -13508,7 +13489,7 @@ "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "integrity": "sha1-lMVA4f93KVbiKZUHwBCupsiDjrA=", "dev": true, "requires": { "punycode": "^2.1.0" @@ -13570,7 +13551,7 @@ "use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", - "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "integrity": "sha1-1QyMrHmhn7wg8pEfVuuXP04QBw8=", "dev": true }, "util": { @@ -14309,7 +14290,7 @@ "websocket-extensions": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", - "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==", + "integrity": "sha1-XS/yKXcAPsaHpLhwc9+7rBRszyk=", "dev": true }, "whatwg-fetch": { @@ -14423,7 +14404,7 @@ "ws": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ws/-/ws-1.1.5.tgz", - "integrity": "sha512-o3KqipXNUdS7wpQzBHSe180lBGO60SoK0yVo3CYJgb2MkobuWuBX6dhkYP5ORCLd55y+SaflMOV5fqAB53ux4w==", + "integrity": "sha1-y9nm514J/F0skAFfIfDECHXg3VE=", "requires": { "options": ">=0.0.5", "ultron": "1.0.x" diff --git a/ui/src/app/api/queue.service.js b/ui/src/app/api/queue.service.js new file mode 100644 index 0000000000..0fe93e5fa6 --- /dev/null +++ b/ui/src/app/api/queue.service.js @@ -0,0 +1,40 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default angular.module('thingsboard.api.queue', []) + .factory('queueService', queueService) + .name; + +/*@ngInject*/ +function queueService($http, $q) { + var service = { + getTenantQueuesByServiceType: getTenantQueuesByServiceType + }; + + return service; + + function getTenantQueuesByServiceType(serviceType, config) { + let deferred = $q.defer(); + let url = '/api/tenant/queues?serviceType=' + serviceType; + + $http.get(url, config).then(function success(data) { + deferred.resolve(data); + }, function fail() { + deferred.reject(); + }); + return deferred.promise; + } +} \ No newline at end of file diff --git a/ui/src/app/components/queue/index.js b/ui/src/app/components/queue/index.js new file mode 100644 index 0000000000..5236603270 --- /dev/null +++ b/ui/src/app/components/queue/index.js @@ -0,0 +1,23 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import thingsboardApiQueue from '../../api/queue.service'; +import queueTypeList from "./queue-type-list.directive"; + +export default angular.module('thingsboard.queue', [ + thingsboardApiQueue +]) + .directive('tbQueueTypeList', queueTypeList) + .name; diff --git a/ui/src/app/components/queue/queue-type-list.directive.js b/ui/src/app/components/queue/queue-type-list.directive.js new file mode 100644 index 0000000000..6a2f99d280 --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.directive.js @@ -0,0 +1,111 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import './queue-type-list.scss'; + +/* eslint-disable import/no-unresolved, import/default */ + +import queueTypeListTemplate from './queue-type-list.tpl.html'; + +/* eslint-enable import/no-unresolved, import/default */ + +/*@ngInject*/ +export default function QueueTypeList($compile, $templateCache, $q, $filter, queueService) { + + var linker = function (scope, element, attrs, ngModelCtrl) { + var template = $templateCache.get(queueTypeListTemplate); + element.html(template); + + scope.queues = null; + scope.queue = null; + scope.tbRequired = angular.isDefined(scope.tbRequired) ? scope.tbRequired : false; + scope.queueSearchText = ''; + + scope.fetchQueues = function(searchText) { + var deferred = $q.defer(); + loadQueues().then( + function success(queueArr) { + let result = $filter('filter')(queueArr, {'$': searchText}); + if (result && result.length) { + if (searchText && searchText.length && result.indexOf(searchText) === -1) { + result.push(searchText); + } + result.sort(); + deferred.resolve(result); + } else { + deferred.resolve([searchText]); + } + }, + function fail() { + deferred.reject(); + } + ); + + return deferred.promise; + }; + + scope.updateView = function () { + if (!scope.disabled) { + ngModelCtrl.$setViewValue(scope.queue); + } + }; + + function loadQueues() { + var deferred = $q.defer(); + if (!scope.queues) { + queueService.getTenantQueuesByServiceType(scope.queueType).then( + function success(queueArr) { + scope.queues = queueArr.data; + deferred.resolve(scope.queues); + }, + function fail() { + deferred.reject(); + } + ); + } else { + deferred.resolve(scope.queues); + } + return deferred.promise; + } + + ngModelCtrl.$render = function () { + scope.queue = ngModelCtrl.$viewValue; + }; + + scope.$watch('queue', function (newValue, prevValue) { + if (!angular.equals(newValue, prevValue)) { + scope.updateView(); + } + }); + + scope.$watch('disabled', function () { + scope.updateView(); + }); + + $compile(element.contents())(scope); + }; + + return { + restrict: "E", + require: "^ngModel", + link: linker, + scope: { + theForm: '=?', + tbRequired: '=?', + disabled:'=ngDisabled', + queueType: '=?' + } + }; +} \ No newline at end of file diff --git a/ui/src/app/components/queue/queue-type-list.scss b/ui/src/app/components/queue/queue-type-list.scss new file mode 100644 index 0000000000..b8f6f58fb2 --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.scss @@ -0,0 +1,15 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/ui/src/app/components/queue/queue-type-list.tpl.html b/ui/src/app/components/queue/queue-type-list.tpl.html new file mode 100644 index 0000000000..5fc700026e --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.tpl.html @@ -0,0 +1,43 @@ + + + +
+ {{item}} +
+ +
+
{{'queue.name_required' | translate}}
+
+ \ No newline at end of file diff --git a/ui/src/app/layout/index.js b/ui/src/app/layout/index.js index a00baa5199..f3b4d10e4f 100644 --- a/ui/src/app/layout/index.js +++ b/ui/src/app/layout/index.js @@ -55,6 +55,7 @@ import thingsboardEntityView from '../entity-view'; import thingsboardWidgetLibrary from '../widget'; import thingsboardDashboard from '../dashboard'; import thingsboardRuleChain from '../rulechain'; +import thingsboardQueue from '../components/queue'; import thingsboardJsonForm from '../jsonform'; @@ -86,6 +87,7 @@ export default angular.module('thingsboard.home', [ thingsboardWidgetLibrary, thingsboardDashboard, thingsboardRuleChain, + thingsboardQueue, thingsboardJsonForm, thingsboardApiDevice, thingsboardApiLogin, diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index 8c7a89580d..fa4689fc75 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1489,6 +1489,12 @@ "help": "Help", "reset-debug-mode": "Reset debug mode in all nodes" }, + "queue": { + "select_name": "Select queue name", + "name": "Queue Name", + "name_required": "Queue name is required!" + + }, "tenant": { "tenant": "Tenant", "tenants": "Tenants", @@ -1514,7 +1520,11 @@ "idCopiedMessage": "Tenant Id has been copied to clipboard", "select-tenant": "Select tenant", "no-tenants-matching": "No tenants matching '{{entity}}' were found.", - "tenant-required": "Tenant is required" + "tenant-required": "Tenant is required", + "isolated-tb-core": "Processing in isolated ThingsBoard Core container", + "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", + "isolated-tb-core-details": "Requires separate microservice(s) per isolated Tenant", + "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", diff --git a/ui/src/app/locale/locale.constant-ru_RU.json b/ui/src/app/locale/locale.constant-ru_RU.json index f86eb988ff..e93c92124c 100644 --- a/ui/src/app/locale/locale.constant-ru_RU.json +++ b/ui/src/app/locale/locale.constant-ru_RU.json @@ -1403,6 +1403,12 @@ "help": "Помощь", "reset-debug-mode": "Сбросить режим отладки во всех правилах" }, + "queue": { + "select_name": "Выберите имя для Queue", + "name": "Имя для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + + }, "tenant": { "tenant": "Владелец", "tenants": "Владельцы", diff --git a/ui/src/app/locale/locale.constant-uk_UA.json b/ui/src/app/locale/locale.constant-uk_UA.json index 5a0899014d..c19049bc11 100644 --- a/ui/src/app/locale/locale.constant-uk_UA.json +++ b/ui/src/app/locale/locale.constant-uk_UA.json @@ -1820,6 +1820,12 @@ "help": "Допомога", "reset-debug-mode": "Вимкнути режим налогодження у всіх правилах" }, + "queue": { + "select_name": "Виберіть ім'я для Queue", + "name": "Iм'я для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + + }, "scheduler": { "scheduler": "Планувальник", "scheduler-event": "Подія планувальника", diff --git a/ui/src/app/tenant/tenant-fieldset.tpl.html b/ui/src/app/tenant/tenant-fieldset.tpl.html index aa7751d479..b75ea30f06 100644 --- a/ui/src/app/tenant/tenant-fieldset.tpl.html +++ b/ui/src/app/tenant/tenant-fieldset.tpl.html @@ -15,32 +15,50 @@ limitations under the License. --> -{{ 'tenant.manage-tenant-admins' | translate }} -{{ 'tenant.delete' | translate }} +{{ + 'tenant.manage-tenant-admins' | translate }} + +{{ 'tenant.delete' + | translate }} +
- - - tenant.copyId - + + + tenant.copyId +
-
- - - -
-
tenant.title-required
-
- - - - - - -
+
+ + + +
+
tenant.title-required
+
+ + + + + + + + + {{'tenant.isolated-tb-core' | translate}}
+ {{'tenant.isolated-tb-core-details' | translate}} + + + + + {{'tenant.isolated-tb-rule-engine' | translate}}
+ {{'tenant.isolated-tb-rule-engine-details' | translate}} + + +