From 8f2438d6ab57ca22b940a001e9a05e68225e6c69 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 22 Mar 2021 17:17:42 +0200 Subject: [PATCH] SNMP devices balancing (#4254) * Fix merge errors * Implement SNMP transports balancing * Refactor; implement transport device cache * Refactor * Finish up device lifecycle handling implementing; refactor * Refactor * Change base image to thingsboard/openjdk11 for msa snmp transport * Refactor * Change transport services names to upper-case --- .../actors/service/DefaultActorService.java | 4 +- .../DefaultTbApiUsageStateService.java | 2 +- .../apiusage/TbApiUsageStateService.java | 2 +- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../DefaultTbRuleEngineConsumerService.java | 2 +- .../service/queue/TbCoreConsumerService.java | 2 +- .../queue/TbRuleEngineConsumerService.java | 2 +- .../processing/AbstractConsumerService.java | 2 +- .../state/DefaultDeviceStateService.java | 2 +- .../service/state/DeviceStateService.java | 2 +- .../DefaultSubscriptionManagerService.java | 2 +- .../DefaultTbLocalSubscriptionService.java | 5 +- .../SubscriptionManagerService.java | 2 +- .../TbLocalSubscriptionService.java | 4 +- .../AbstractSubscriptionService.java | 19 +- .../telemetry/AlarmSubscriptionService.java | 3 +- .../DefaultAlarmSubscriptionService.java | 23 - .../TelemetrySubscriptionService.java | 3 +- .../transport/DefaultTransportApiService.java | 94 ++++- .../server/dao/device/DeviceService.java | 3 + .../common/data/TbTransportService.java | 20 + .../data/DeviceTransportConfiguration.java | 4 +- .../SnmpProfileTransportConfiguration.java | 10 +- .../DefaultTbServiceInfoProvider.java | 20 + .../queue/discovery/HashPartitionService.java | 20 +- .../queue/discovery/PartitionService.java | 4 + .../discovery/TbApplicationEventListener.java | 1 + .../queue/discovery/ZkDiscoveryService.java | 12 +- .../ClusterTopologyChangeEvent.java | 3 +- .../{ => event}/PartitionChangeEvent.java | 3 +- .../event/ServiceListChangedEvent.java | 35 ++ .../{ => event}/TbApplicationEvent.java | 2 +- .../queue/util/TbSnmpTransportComponent.java | 29 ++ common/queue/src/main/proto/queue.proto | 35 ++ .../transport/coap/CoapTransportService.java | 9 +- .../transport/http/DeviceApiController.java | 10 +- .../lwm2m/server/LwM2mTransportService.java | 3 +- .../server/LwM2mTransportServiceImpl.java | 6 + .../transport/mqtt/MqttTransportService.java | 8 +- common/transport/snmp/pom.xml | 4 - .../transport/snmp/SnmpDeviceSimulator.java | 81 ++++ .../transport/snmp/SnmpTransportContext.java | 311 +++++++++++--- .../transport/snmp/SnmpTransportService.java | 147 ------- .../ServiceListChangedEventListener.java | 35 ++ .../event/SnmpTransportListChangedEvent.java | 24 ++ ...SnmpTransportListChangedEventListener.java | 34 ++ .../service/ProtoTransportEntityService.java | 92 ++++ .../SnmpTransportBalancingService.java | 92 ++++ .../SnmpTransportService.java} | 149 +++++-- .../snmp/session/DeviceSessionContext.java | 166 ++++++++ .../snmp/session/DeviceSessionCtx.java | 206 --------- .../common/transport/DeviceUpdatedEvent.java | 28 ++ .../common/transport/SessionMsgListener.java | 4 + .../common/transport/TransportContext.java | 1 + .../common/transport/TransportService.java | 12 + .../DefaultTransportDeviceProfileCache.java | 4 +- .../service/DefaultTransportService.java | 77 +++- .../server/dao/device/DeviceDao.java | 4 + .../server/dao/device/DeviceServiceImpl.java | 7 + .../dao/sql/device/DeviceRepository.java | 6 + .../server/dao/sql/device/JpaDeviceDao.java | 6 + docker/tb-transports/snmp/conf/logback.xml | 50 +++ .../snmp/conf/tb-snmp-transport.conf | 23 + msa/transport/snmp/docker/Dockerfile | 2 +- transport/snmp/src/main/conf/logback.xml | 45 ++ .../snmp/src/main/conf/tb-snmp-transport.conf | 22 + transport/snmp/src/main/resources/logback.xml | 36 ++ .../src/main/resources/tb-snmp-transport.yml | 394 ++++++++++++++++++ 68 files changed, 1923 insertions(+), 553 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/TbTransportService.java rename common/queue/src/main/java/org/thingsboard/server/queue/discovery/{ => event}/ClusterTopologyChangeEvent.java (91%) rename common/queue/src/main/java/org/thingsboard/server/queue/discovery/{ => event}/PartitionChangeEvent.java (93%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ServiceListChangedEvent.java rename common/queue/src/main/java/org/thingsboard/server/queue/discovery/{ => event}/TbApplicationEvent.java (95%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbSnmpTransportComponent.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java delete mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/ServiceListChangedEventListener.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEvent.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEventListener.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportBalancingService.java rename common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/{session/SnmpSessionListener.java => service/SnmpTransportService.java} (60%) create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java delete mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/DeviceUpdatedEvent.java create mode 100644 docker/tb-transports/snmp/conf/logback.xml create mode 100644 docker/tb-transports/snmp/conf/tb-snmp-transport.conf create mode 100644 transport/snmp/src/main/conf/logback.xml create mode 100644 transport/snmp/src/main/conf/tb-snmp-transport.conf create mode 100644 transport/snmp/src/main/resources/logback.xml create mode 100644 transport/snmp/src/main/resources/tb-snmp-transport.yml diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 05363dfd59..f3a07eb727 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -25,7 +25,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.DefaultTbActorSystem; -import org.thingsboard.server.actors.TbActorId; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbActorSystem; import org.thingsboard.server.actors.TbActorSystemSettings; @@ -33,14 +32,13 @@ import org.thingsboard.server.actors.app.AppActor; import org.thingsboard.server.actors.app.AppInitMsg; import org.thingsboard.server.actors.stats.StatsActor; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; @Service @Slf4j diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index d0a3984660..ccac9b4a16 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -52,7 +52,7 @@ import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.scheduler.SchedulerComponent; diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java index 117607ae9f..f97188597b 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; public interface TbApiUsageStateService extends ApplicationListener { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 02f69ce10d..9d3e2b5a64 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -52,7 +52,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 390798a3e2..bd29b6c431 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -38,7 +38,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java index 0a965413cf..222f58fd78 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -16,7 +16,7 @@ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; public interface TbCoreConsumerService extends ApplicationListener { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java index e4a88de575..582dbd1355 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -16,7 +16,7 @@ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; public interface TbRuleEngineConsumerService extends ApplicationListener { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 02378eb557..28cce50bf5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -34,7 +34,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index c8a4e80e0e..5c358c1b77 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -54,7 +54,7 @@ import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.util.TbCoreComponent; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 932ed4720d..b1bbb9cdfa 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -18,7 +18,7 @@ package org.thingsboard.server.service.state; import org.springframework.context.ApplicationListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.common.msg.queue.TbCallback; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index ce8838aaa8..657bdf6f75 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -46,7 +46,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdate import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateValueListProto; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 0220a94964..710d613b37 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -20,10 +20,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java index fd40364fe5..37850b701f 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java index d244ef136b..f1bf6c0da0 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java @@ -15,8 +15,8 @@ */ package org.thingsboard.server.service.subscription; -import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate; import org.thingsboard.server.service.telemetry.sub.TelemetrySubscriptionUpdate; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index 168e1271fd..23abba15f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -22,35 +22,18 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; -import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.Collections; -import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AlarmSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AlarmSubscriptionService.java index fa44906cc3..6ed3860341 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AlarmSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AlarmSubscriptionService.java @@ -17,8 +17,7 @@ package org.thingsboard.server.service.telemetry; import org.springframework.context.ApplicationListener; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; -import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; /** * Created by ashvayka on 27.03.18. diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java index 107b97dbb3..dde71ef53f 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java @@ -22,9 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; @@ -35,43 +33,22 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmData; -import org.thingsboard.server.common.data.query.AlarmDataPageLink; import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.alarm.AlarmOperationResult; import org.thingsboard.server.dao.alarm.AlarmService; -import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; -import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.Collection; -import java.util.Collections; -import java.util.List; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Consumer; /** * Created by ashvayka on 27.03.18. diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index 015a58495f..0ac9e79ff1 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -16,8 +16,7 @@ package org.thingsboard.server.service.telemetry; import org.springframework.context.ApplicationListener; -import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; /** * Created by ashvayka on 27.03.18. diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index eea7af582c..4f58bd0f63 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; @@ -61,11 +62,15 @@ import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceCredentialsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetResourcesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionResponseStatus; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; @@ -84,6 +89,7 @@ import org.thingsboard.server.service.state.DeviceStateService; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -139,40 +145,41 @@ public class DefaultTransportApiService implements TransportApiService { @Override public ListenableFuture> handle(TbProtoQueueMsg tbProtoQueueMsg) { TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue(); + ListenableFuture result = null; + if (transportApiRequestMsg.hasValidateTokenRequestMsg()) { ValidateDeviceTokenRequestMsg msg = transportApiRequestMsg.getValidateTokenRequestMsg(); - return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN); } else if (transportApiRequestMsg.hasValidateBasicMqttCredRequestMsg()) { TransportProtos.ValidateBasicMqttCredRequestMsg msg = transportApiRequestMsg.getValidateBasicMqttCredRequestMsg(); - return Futures.transform(validateCredentials(msg), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = validateCredentials(msg); } else if (transportApiRequestMsg.hasValidateX509CertRequestMsg()) { ValidateDeviceX509CertRequestMsg msg = transportApiRequestMsg.getValidateX509CertRequestMsg(); - return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE); } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()); } else if (transportApiRequestMsg.hasEntityProfileRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getEntityProfileRequestMsg()), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = handle(transportApiRequestMsg.getEntityProfileRequestMsg()); } else if (transportApiRequestMsg.hasLwM2MRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getLwM2MRequestMsg()), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = handle(transportApiRequestMsg.getLwM2MRequestMsg()); } else if (transportApiRequestMsg.hasValidateDeviceLwM2MCredentialsRequestMsg()) { ValidateDeviceLwM2MCredentialsRequestMsg msg = transportApiRequestMsg.getValidateDeviceLwM2MCredentialsRequestMsg(); - return Futures.transform(validateCredentials(msg.getCredentialsId(), DeviceCredentialsType.LWM2M_CREDENTIALS), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = validateCredentials(msg.getCredentialsId(), DeviceCredentialsType.LWM2M_CREDENTIALS); } else if (transportApiRequestMsg.hasProvisionDeviceRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getProvisionDeviceRequestMsg()), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = handle(transportApiRequestMsg.getProvisionDeviceRequestMsg()); } else if (transportApiRequestMsg.hasResourcesRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getResourcesRequestMsg()), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = handle(transportApiRequestMsg.getResourcesRequestMsg()); + } else if (transportApiRequestMsg.hasSnmpDevicesRequestMsg()) { + result = handle(transportApiRequestMsg.getSnmpDevicesRequestMsg()); + } else if (transportApiRequestMsg.hasDeviceRequestMsg()) { + result = handle(transportApiRequestMsg.getDeviceRequestMsg()); + } else if (transportApiRequestMsg.hasDeviceCredentialsRequestMsg()) { + result = handle(transportApiRequestMsg.getDeviceCredentialsRequestMsg()); } - return Futures.transform(getEmptyTransportApiResponseFuture(), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + + return Futures.transform(Optional.ofNullable(result).orElseGet(this::getEmptyTransportApiResponseFuture), + value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), + MoreExecutors.directExecutor()); } private ListenableFuture validateCredentials(String credentialsId, DeviceCredentialsType credentialsType) { @@ -366,6 +373,39 @@ public class DefaultTransportApiService implements TransportApiService { return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(builder).build()); } + private ListenableFuture handle(GetDeviceRequestMsg requestMsg) { + DeviceId deviceId = new DeviceId(new UUID(requestMsg.getDeviceIdMSB(), requestMsg.getDeviceIdLSB())); + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + + TransportApiResponseMsg responseMsg; + if (device != null) { + UUID deviceProfileId = device.getDeviceProfileId().getId(); + responseMsg = TransportApiResponseMsg.newBuilder() + .setDeviceResponseMsg(TransportProtos.GetDeviceResponseMsg.newBuilder() + .setDeviceProfileIdMSB(deviceProfileId.getMostSignificantBits()) + .setDeviceProfileIdLSB(deviceProfileId.getLeastSignificantBits()) + .setDeviceTransportConfiguration(ByteString.copyFrom( + dataDecodingEncodingService.encode(device.getDeviceData().getTransportConfiguration()) + ))) + .build(); + } else { + responseMsg = TransportApiResponseMsg.getDefaultInstance(); + } + + return Futures.immediateFuture(responseMsg); + } + + private ListenableFuture handle(GetDeviceCredentialsRequestMsg requestMsg) { + DeviceId deviceId = new DeviceId(new UUID(requestMsg.getDeviceIdMSB(), requestMsg.getDeviceIdLSB())); + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(TenantId.SYS_TENANT_ID, deviceId); + + return Futures.immediateFuture(TransportApiResponseMsg.newBuilder() + .setDeviceCredentialsResponseMsg(TransportProtos.GetDeviceCredentialsResponseMsg.newBuilder() + .setDeviceCredentialsData(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceCredentials)))) + .build()); + } + + private ListenableFuture handle(GetResourcesRequestMsg requestMsg) { TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB())); TransportProtos.GetResourcesResponseMsg.Builder builder = TransportProtos.GetResourcesResponseMsg.newBuilder(); @@ -400,6 +440,20 @@ public class DefaultTransportApiService implements TransportApiService { .build(); } + // TODO: request snmp devices with pagination + private ListenableFuture handle(GetSnmpDevicesRequestMsg requestMsg) { + List result = deviceService.findDevicesIdsByDeviceProfileTransportType(DeviceTransportType.SNMP); + GetSnmpDevicesResponseMsg responseMsg = GetSnmpDevicesResponseMsg.newBuilder() + .addAllIds(result.stream() + .map(UUID::toString) + .collect(Collectors.toList())) + .build(); + + return Futures.immediateFuture(TransportApiResponseMsg.newBuilder() + .setSnmpDevicesResponseMsg(responseMsg) + .build()); + } + private ListenableFuture getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { return Futures.transform(deviceService.findDeviceByIdAsync(TenantId.SYS_TENANT_ID, deviceId), device -> { if (device == null) { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index f98c56d89a..1a893f5388 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.id.CustomerId; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.dao.device.provision.ProvisionRequest; import java.util.List; +import java.util.UUID; public interface DeviceService { @@ -90,4 +92,5 @@ public interface DeviceService { Device saveDevice(ProvisionRequest provisionRequest, DeviceProfile profile); + List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TbTransportService.java b/common/data/src/main/java/org/thingsboard/server/common/data/TbTransportService.java new file mode 100644 index 0000000000..6195022aba --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TbTransportService.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +public interface TbTransportService { + String getName(); +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java index 0ae9f26cf3..49a547371f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java @@ -21,6 +21,8 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.DeviceTransportType; +import java.io.Serializable; + @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -31,7 +33,7 @@ import org.thingsboard.server.common.data.DeviceTransportType; @JsonSubTypes.Type(value = MqttDeviceTransportConfiguration.class, name = "MQTT"), @JsonSubTypes.Type(value = Lwm2mDeviceTransportConfiguration.class, name = "LWM2M"), @JsonSubTypes.Type(value = SnmpDeviceTransportConfiguration.class, name = "SNMP")}) -public interface DeviceTransportConfiguration { +public interface DeviceTransportConfiguration extends Serializable { @JsonIgnore DeviceTransportType getType(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java index a2529e74e9..50f6c4351c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java @@ -19,14 +19,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.DeviceTransportType; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @Data public class SnmpProfileTransportConfiguration implements DeviceProfileTransportConfiguration { - - private int poolPeriodMs; + private int pollPeriodMs; private int timeoutMs; private int retries; private List attributes; @@ -39,6 +39,10 @@ public class SnmpProfileTransportConfiguration implements DeviceProfileTransport @JsonIgnore public List getKvMappings() { - return Stream.concat(attributes.stream(), telemetry.stream()).collect(Collectors.toList()); + if (attributes != null && telemetry != null) { + return Stream.concat(attributes.stream(), telemetry.stream()).collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index fc4ec33eb1..3ba360cd8e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -19,8 +19,12 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationContext; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; @@ -32,6 +36,7 @@ import javax.annotation.PostConstruct; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -56,6 +61,8 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Autowired(required = false) private TbQueueRuleEngineSettings ruleEngineSettings; + @Autowired + private ApplicationContext applicationContext; private List serviceTypes; private ServiceInfo serviceInfo; @@ -102,6 +109,19 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { serviceInfo = builder.build(); } + @EventListener(ContextRefreshedEvent.class) + public void setTransports() { + serviceInfo = ServiceInfo.newBuilder(serviceInfo) + .addAllTransports(getTransportServices().stream() + .map(TbTransportService::getName) + .collect(Collectors.toSet())) + .build(); + } + + private Collection getTransportServices() { + return applicationContext.getBeansOfType(TbTransportService.class).values(); + } + @Override public ServiceInfo getServiceInfo() { return serviceInfo; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 2da438417a..c833b314e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -15,26 +15,27 @@ */ package org.thingsboard.server.queue.discovery; -import com.google.common.hash.HashCode; import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceQueueKey; import org.thingsboard.server.common.msg.queue.ServiceQueue; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import javax.annotation.PostConstruct; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -46,7 +47,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentNavigableMap; import java.util.stream.Collectors; @Service @@ -186,6 +186,8 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); } } + + applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } @Override @@ -219,6 +221,14 @@ public class HashPartitionService implements PartitionService { } } + @Override + public int resolvePartitionIndex(UUID entityId, int partitions) { + int hash = hashFunction.newHasher() + .putLong(entityId.getMostSignificantBits()) + .putLong(entityId.getLeastSignificantBits()).hash().asInt(); + return Math.abs(hash % partitions); + } + private Map> getServiceKeyListMap(List services) { final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index ebb260bc57..20c59378e8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -20,9 +20,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import java.util.List; import java.util.Set; +import java.util.UUID; /** * Once application is ready or cluster topology changes, this Service will produce {@link PartitionChangeEvent} @@ -55,4 +57,6 @@ public interface PartitionService { * @return */ TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId); + + int resolvePartitionIndex(UUID entityId, int partitions); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java index 9158d8f0c8..70484a892b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.discovery; import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.event.TbApplicationEvent; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 5253f57248..0802f0f52c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -33,12 +33,14 @@ import org.apache.zookeeper.KeeperException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -77,7 +79,8 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi private volatile boolean stopped = true; - public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider, + PartitionService partitionService) { this.serviceInfoProvider = serviceInfoProvider; this.partitionService = partitionService; } @@ -126,7 +129,8 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return; } publishCurrentServer(); - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); + TransportProtos.ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); + partitionService.recalculatePartitions(currentService, getOtherServers()); } public synchronized void publishCurrentServer() { @@ -281,11 +285,11 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi case CHILD_ADDED: case CHILD_UPDATED: case CHILD_REMOVED: - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); + TransportProtos.ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); + partitionService.recalculatePartitions(currentService, getOtherServers()); break; default: break; } } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ClusterTopologyChangeEvent.java similarity index 91% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ClusterTopologyChangeEvent.java index 1e5b90b5fe..6b3a3ff93f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ClusterTopologyChangeEvent.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.queue.discovery.event; import lombok.Getter; -import org.springframework.context.ApplicationEvent; import org.thingsboard.server.common.msg.queue.ServiceQueueKey; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java similarity index 93% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 2edcd2ceca..e11e19db61 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.queue.discovery.event; import lombok.Getter; -import org.springframework.context.ApplicationEvent; import org.thingsboard.server.common.msg.queue.ServiceQueueKey; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ServiceListChangedEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ServiceListChangedEvent.java new file mode 100644 index 0000000000..dd7d384640 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ServiceListChangedEvent.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery.event; + +import lombok.Getter; +import lombok.ToString; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; + +import java.util.List; + +@Getter +@ToString +public class ServiceListChangedEvent extends TbApplicationEvent { + private final List otherServices; + private final ServiceInfo currentService; + + public ServiceListChangedEvent(List otherServices, ServiceInfo currentService) { + super(otherServices); + this.otherServices = otherServices; + this.currentService = currentService; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/TbApplicationEvent.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/TbApplicationEvent.java index face2d36d6..1a75b4e4e4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/TbApplicationEvent.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.queue.discovery.event; import lombok.Getter; import org.springframework.context.ApplicationEvent; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbSnmpTransportComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbSnmpTransportComponent.java new file mode 100644 index 0000000000..36080afaab --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbSnmpTransportComponent.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.snmp.enabled}'=='true')") +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.METHOD}) +public @interface TbSnmpTransportComponent { +} diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 42f6267d42..59fcdad36a 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -34,6 +34,7 @@ message ServiceInfo { int64 tenantIdMSB = 3; int64 tenantIdLSB = 4; repeated QueueInfo ruleEngineQueues = 5; + repeated string transports = 6; } /** @@ -250,6 +251,34 @@ message GetEntityProfileResponseMsg { bytes apiState = 3; } +message GetDeviceRequestMsg { + int64 deviceIdMSB = 1; + int64 deviceIdLSB = 2; +} + +message GetDeviceResponseMsg { + int64 deviceProfileIdMSB = 1; + int64 deviceProfileIdLSB = 2; + bytes deviceTransportConfiguration = 3; +} + +message GetDeviceCredentialsRequestMsg { + int64 deviceIdMSB = 1; + int64 deviceIdLSB = 2; +} + +message GetDeviceCredentialsResponseMsg { + bytes deviceCredentialsData = 1; +} + +message GetSnmpDevicesRequestMsg { + +} + +message GetSnmpDevicesResponseMsg { + repeated string ids = 1; +} + message EntityUpdateMsg { string entityType = 1; bytes data = 2; @@ -559,6 +588,9 @@ message TransportApiRequestMsg { ProvisionDeviceRequestMsg provisionDeviceRequestMsg = 7; ValidateDeviceLwM2MCredentialsRequestMsg validateDeviceLwM2MCredentialsRequestMsg = 8; GetResourcesRequestMsg resourcesRequestMsg = 9; + GetSnmpDevicesRequestMsg snmpDevicesRequestMsg = 10; + GetDeviceRequestMsg deviceRequestMsg = 11; + GetDeviceCredentialsRequestMsg deviceCredentialsRequestMsg = 12; } /* Response from ThingsBoard Core Service to Transport Service */ @@ -567,8 +599,11 @@ message TransportApiResponseMsg { GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; GetEntityProfileResponseMsg entityProfileResponseMsg = 3; ProvisionDeviceResponseMsg provisionDeviceResponseMsg = 4; + GetSnmpDevicesResponseMsg snmpDevicesResponseMsg = 5; LwM2MResponseMsg lwM2MResponseMsg = 6; GetResourcesResponseMsg resourcesResponseMsg = 7; + GetDeviceResponseMsg deviceResponseMsg = 8; + GetDeviceCredentialsResponseMsg deviceCredentialsResponseMsg = 9; } /* Messages that are handled by ThingsBoard Core Service */ diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index 8abc859b21..dd4a76dcc8 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -18,12 +18,12 @@ package org.thingsboard.server.transport.coap; import lombok.extern.slf4j.Slf4j; import org.eclipse.californium.core.CoapResource; import org.eclipse.californium.core.CoapServer; - import org.eclipse.californium.core.network.CoapEndpoint; import org.eclipse.californium.core.network.CoapEndpoint.Builder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.TbTransportService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -34,7 +34,7 @@ import java.net.UnknownHostException; @Service("CoapTransportService") @ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.coap.enabled}'=='true')") @Slf4j -public class CoapTransportService { +public class CoapTransportService implements TbTransportService { private static final String V1 = "v1"; private static final String API = "api"; @@ -73,4 +73,9 @@ public class CoapTransportService { this.server.destroy(); log.info("CoAP transport stopped!"); } + + @Override + public String getName() { + return "COAP"; + } } diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index e222b690ef..011093149e 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -31,6 +31,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportContext; @@ -41,7 +42,6 @@ import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; @@ -53,7 +53,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMs import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import javax.servlet.http.HttpServletRequest; @@ -69,7 +68,7 @@ import java.util.function.Consumer; @ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.http.enabled}'=='true')") @RequestMapping("/api/v1") @Slf4j -public class DeviceApiController { +public class DeviceApiController implements TbTransportService { @Autowired private HttpTransportContext transportContext; @@ -338,4 +337,9 @@ public class DeviceApiController { .build(), TransportServiceCallback.EMPTY); } + @Override + public String getName() { + return "HTTP"; + } + } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java index 8d1aff37b5..db419eca9f 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java @@ -20,12 +20,13 @@ import org.eclipse.leshan.core.response.ReadResponse; import org.eclipse.leshan.server.registration.Registration; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.Collection; import java.util.Optional; -public interface LwM2mTransportService { +public interface LwM2mTransportService extends TbTransportService { void onRegistered(Registration registration, Collection previousObsersations); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java index 548d750a3f..bae6a0af1d 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java @@ -40,6 +40,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; @@ -1118,4 +1119,9 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { namesIsWritable.addAll(new HashSet<>(keyNamesIsWritable.values())); return new ArrayList<>(namesIsWritable); } + + @Override + public String getName() { + return "LWM2M"; + } } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java index 0a0c0cbe70..3e53dec227 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java @@ -28,6 +28,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.TbTransportService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -38,7 +39,7 @@ import javax.annotation.PreDestroy; @Service("MqttTransportService") @ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.mqtt.enabled}'=='true')") @Slf4j -public class MqttTransportService { +public class MqttTransportService implements TbTransportService { @Value("${transport.mqtt.bind_address}") private String host; @@ -90,4 +91,9 @@ public class MqttTransportService { } log.info("MQTT transport stopped!"); } + + @Override + public String getName() { + return "MQTT"; + } } diff --git a/common/transport/snmp/pom.xml b/common/transport/snmp/pom.xml index c7e22ecc80..ab83ab69a7 100644 --- a/common/transport/snmp/pom.xml +++ b/common/transport/snmp/pom.xml @@ -58,9 +58,5 @@ org.snmp4j snmp4j - - org.thingsboard.common - dao-api - diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java new file mode 100644 index 0000000000..7109ef4e3f --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp; + +import org.snmp4j.CommunityTarget; +import org.snmp4j.PDU; +import org.snmp4j.Snmp; +import org.snmp4j.Target; +import org.snmp4j.event.ResponseEvent; +import org.snmp4j.mp.SnmpConstants; +import org.snmp4j.smi.GenericAddress; +import org.snmp4j.smi.OID; +import org.snmp4j.smi.OctetString; +import org.snmp4j.smi.VariableBinding; +import org.snmp4j.transport.DefaultUdpTransportMapping; +import org.snmp4j.transport.UdpTransportMapping; + +import java.io.IOException; + +/** + * For testing purposes. Will be removed when the time comes + */ +public class SnmpDeviceSimulator { + private final Target target; + private final OID oid = new OID(".1.3.6.1.2.1.1.1.0"); + private Snmp snmp; + + public SnmpDeviceSimulator(int port) { + String address = "udp:127.0.0.1/" + port; + + CommunityTarget target = new CommunityTarget(); + target.setCommunity(new OctetString("public")); + target.setAddress(GenericAddress.parse(address)); + target.setRetries(2); + target.setTimeout(1500); + target.setVersion(SnmpConstants.version2c); + + this.target = target; + } + + public static void main(String[] args) throws IOException { + SnmpDeviceSimulator deviceSimulator = new SnmpDeviceSimulator(161); + + deviceSimulator.start(); + String response = deviceSimulator.sendRequest(PDU.GET); + + System.out.println(response); + } + + public void start() throws IOException { + UdpTransportMapping transport = new DefaultUdpTransportMapping(); + transport.addTransportListener((sourceTransport, incomingAddress, wholeMessage, tmStateReference) -> { + System.out.println(); + }); + snmp = new Snmp(transport); + + transport.listen(); + } + + public String sendRequest(int pduType) throws IOException { + PDU pdu = new PDU(); + pdu.add(new VariableBinding(oid)); + pdu.setType(pduType); + + ResponseEvent responseEvent = snmp.send(pdu, target); + return responseEvent.getResponse().get(0).getVariable().toString(); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index 66f852a209..a9fa8380fa 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -15,17 +15,19 @@ */ package org.thingsboard.server.transport.snmp; -import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; -import org.snmp4j.Snmp; import org.snmp4j.smi.OID; import org.snmp4j.smi.VariableBinding; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileKvMapping; import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; @@ -33,84 +35,180 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import org.thingsboard.server.common.transport.DeviceUpdatedEvent; import org.thingsboard.server.common.transport.TransportContext; -import org.thingsboard.server.dao.device.DeviceCredentialsService; -import org.thingsboard.server.transport.snmp.session.DeviceSessionCtx; +import org.thingsboard.server.common.transport.TransportDeviceProfileCache; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.common.transport.auth.SessionInfoCreator; +import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; +import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.service.ProtoTransportEntityService; +import org.thingsboard.server.transport.snmp.service.SnmpTransportBalancingService; +import org.thingsboard.server.transport.snmp.service.SnmpTransportService; +import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ExecutorService; import java.util.stream.Collectors; -@Service("SnmpTransportContext") -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.snmp.enabled}'=='true')") +@TbSnmpTransportComponent +@Component @Slf4j +@RequiredArgsConstructor public class SnmpTransportContext extends TransportContext { - @Autowired - DeviceCredentialsService deviceCredentialsService; - - @Autowired - SnmpTransportService snmpTransportService; - - @Getter - private final Map profileTransportConfig = new ConcurrentHashMap<>(); - @Getter - private final Map> pdusPerProfile = new ConcurrentHashMap<>(); - @Getter - private final Map deviceSessions = new ConcurrentHashMap<>(); - - public Optional findAttributesMapping(DeviceProfileId deviceProfileId, OID responseOid) { - if (profileTransportConfig.containsKey(deviceProfileId)) { - return findMapping(responseOid, profileTransportConfig.get(deviceProfileId).getAttributes()); - } - return Optional.empty(); + private final SnmpTransportService snmpTransportService; + private final TransportDeviceProfileCache deviceProfileCache; + private final TransportService transportService; + private final ProtoTransportEntityService protoEntityService; + private final SnmpTransportBalancingService balancingService; + + private final Map sessions = new ConcurrentHashMap<>(); + private final Map profilesTransportConfigs = new ConcurrentHashMap<>(); + private final Map> profilesPdus = new ConcurrentHashMap<>(); + private Collection allSnmpDevicesIds = new ConcurrentLinkedDeque<>(); + + @EventListener(ApplicationReadyEvent.class) + @Order(2) + public void initDevicesSessions() { + log.info("Initializing SNMP devices sessions"); + allSnmpDevicesIds = protoEntityService.getAllSnmpDevicesIds().stream() + .map(DeviceId::new) + .collect(Collectors.toList()); + log.trace("Found all SNMP devices ids: {}", allSnmpDevicesIds); + + List managedDevicesIds = allSnmpDevicesIds.stream() + .filter(deviceId -> balancingService.isManagedByCurrentTransport(deviceId.getId())) + .collect(Collectors.toList()); + log.info("SNMP devices managed by current SNMP transport: {}", managedDevicesIds); + + managedDevicesIds.stream() + .map(protoEntityService::getDeviceById) + .collect(Collectors.toList()) + .forEach(this::establishDeviceSession); } - public Optional findTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) { - if (profileTransportConfig.containsKey(deviceProfileId)) { - return findMapping(responseOid, profileTransportConfig.get(deviceProfileId).getTelemetry()); + private void establishDeviceSession(Device device) { + if (device == null) return; + log.info("Establishing SNMP device session for device {}", device.getId()); + + DeviceProfileId deviceProfileId = device.getDeviceProfileId(); + DeviceProfile deviceProfile = deviceProfileCache.get(deviceProfileId); + + DeviceCredentials credentials = protoEntityService.getDeviceCredentialsByDeviceId(device.getId()); + if (credentials.getCredentialsType() != DeviceCredentialsType.ACCESS_TOKEN) { + log.warn("[{}] Expected credentials type is {} but found {}", device.getId(), DeviceCredentialsType.ACCESS_TOKEN, credentials.getCredentialsType()); + return; } - return Optional.empty(); - } - private Optional findMapping(OID responseOid, List mappings) { - return mappings.stream() - .filter(kvMapping -> new OID(kvMapping.getOid()).equals(responseOid)) - //TODO: OID shouldn't be duplicated in the config, add backend and UI verification - .findFirst(); - } + SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); + if (!deviceTransportConfiguration.isValid()) { + log.warn("SNMP device transport configuration is not valid"); + return; + } - public void initPduListPerProfile() { - profileTransportConfig.forEach(this::updatePduListPerProfile); - } + SnmpProfileTransportConfiguration profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + profilesPdus.computeIfAbsent(deviceProfileId, id -> createPdus(profileTransportConfiguration)); - public void updatePduListPerProfile(DeviceProfileId id, SnmpProfileTransportConfiguration config) { - pdusPerProfile.put(id, createPduList(config)); + DeviceSessionContext deviceSessionContext = new DeviceSessionContext( + device, deviceProfile, + credentials.getCredentialsId(), deviceTransportConfiguration, + this, snmpTransportService + ); + registerSessionMsgListener(deviceSessionContext); + sessions.put(device.getId(), deviceSessionContext); + log.info("Established SNMP device session for device {}", device.getId()); } - public void updateDeviceSessionCtx(Device device, DeviceProfile deviceProfile, Snmp snmp) { - DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(device.getTenantId(), device.getId()); - if (DeviceCredentialsType.ACCESS_TOKEN.equals(credentials.getCredentialsType())) { - SnmpDeviceTransportConfiguration snmpDeviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); - if (snmpDeviceTransportConfiguration.isValid()) { - DeviceSessionCtx deviceSessionCtx = new DeviceSessionCtx(this, credentials.getCredentialsId(), snmpDeviceTransportConfiguration, snmp, device.getId(), deviceProfile); - deviceSessionCtx.createSessionInfo(ctx -> getTransportService().registerAsyncSession(deviceSessionCtx.getSessionInfo(), deviceSessionCtx)); - this.deviceSessions.put(device.getId(), deviceSessionCtx); - } - } else { + private void updateDeviceSession(DeviceSessionContext sessionContext, Device device, DeviceProfile deviceProfile) { + log.info("Updating SNMP device session for device {}", device.getId()); + DeviceProfileId deviceProfileId = deviceProfile.getId(); + + DeviceCredentials credentials = protoEntityService.getDeviceCredentialsByDeviceId(device.getId()); + if (credentials.getCredentialsType() != DeviceCredentialsType.ACCESS_TOKEN) { log.warn("[{}] Expected credentials type is {} but found {}", device.getId(), DeviceCredentialsType.ACCESS_TOKEN, credentials.getCredentialsType()); + destroyDeviceSession(sessionContext); + return; + } + + SnmpProfileTransportConfiguration profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); + sessionContext.setProfileTransportConfiguration(profileTransportConfiguration); + sessionContext.setDeviceTransportConfiguration(deviceTransportConfiguration); + if (!deviceTransportConfiguration.isValid()) { + log.warn("SNMP device transport configuration is not valid"); + destroyDeviceSession(sessionContext); + return; + } + + if (!profileTransportConfiguration.equals(profilesTransportConfigs.get(deviceProfileId))) { + profilesPdus.put(deviceProfileId, createPdus(profileTransportConfiguration)); + profilesTransportConfigs.put(deviceProfileId, profileTransportConfiguration); + sessionContext.initTarget(profileTransportConfiguration, deviceTransportConfiguration); + } else if (!deviceTransportConfiguration.equals(sessionContext.getDeviceTransportConfiguration())) { + sessionContext.initTarget(profileTransportConfiguration, deviceTransportConfiguration); + } else { + log.trace("Configuration of the device {} was not updated", device); } } - public ExecutorService getSnmpCallbackExecutor() { - return snmpTransportService.getSnmpCallbackExecutor(); + private void destroyDeviceSession(DeviceSessionContext sessionContext) { + if (sessionContext == null) return; + log.info("Destroying SNMP device session for device {}", sessionContext.getDevice().getId()); + sessionContext.close(); + transportService.deregisterSession(sessionContext.getSessionInfo()); + sessions.remove(sessionContext.getDeviceId()); + log.trace("Deregistered and removed session"); + + DeviceProfileId deviceProfileId = sessionContext.getDeviceProfile().getId(); + if (sessions.values().stream() + .map(DeviceAwareSessionContext::getDeviceProfile) + .noneMatch(deviceProfile -> deviceProfile.getId().equals(deviceProfileId))) { + log.trace("Removed values for device profile {} from configs and pdus caches", deviceProfileId); + profilesTransportConfigs.remove(deviceProfileId); + profilesPdus.remove(deviceProfileId); + } } - private List createPduList(SnmpProfileTransportConfiguration deviceProfileConfig) { + private void registerSessionMsgListener(DeviceSessionContext deviceSessionContext) { + transportService.process(DeviceTransportType.SNMP, + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceSessionContext.getToken()).build(), + new TransportServiceCallback() { + @Override + public void onSuccess(ValidateDeviceCredentialsResponse msg) { + if (msg.hasDeviceInfo()) { + SessionInfoProto sessionInfo = SessionInfoCreator.create( + msg, SnmpTransportContext.this, UUID.randomUUID() + ); + + transportService.registerAsyncSession(sessionInfo, deviceSessionContext); + deviceSessionContext.setSessionInfo(sessionInfo); + deviceSessionContext.setDeviceInfo(msg.getDeviceInfo()); + } else { + log.warn("[{}] Failed to process device auth", deviceSessionContext.getDeviceId()); + } + } + + @Override + public void onError(Throwable e) { + log.warn("[{}] Failed to process device auth: {}", deviceSessionContext.getDeviceId(), e); + } + }); + } + + private List createPdus(SnmpProfileTransportConfiguration deviceProfileConfig) { Map> varBindingPerMethod = new HashMap<>(); deviceProfileConfig.getKvMappings().forEach(mapping -> varBindingPerMethod @@ -127,7 +225,106 @@ public class SnmpTransportContext extends TransportContext { .collect(Collectors.toList()); } - //TODO: Extract SNMP methods to enum + @EventListener(DeviceUpdatedEvent.class) + public void onDeviceUpdatedOrCreated(DeviceUpdatedEvent deviceUpdatedEvent) { + Device device = deviceUpdatedEvent.getDevice(); + log.trace("Got creating or updating device event for device {}", device); + DeviceTransportType transportType = Optional.ofNullable(device.getDeviceData().getTransportConfiguration()) + .map(DeviceTransportConfiguration::getType) + .orElse(null); + if (!allSnmpDevicesIds.contains(device.getId())) { + if (transportType != DeviceTransportType.SNMP) { + return; + } + allSnmpDevicesIds.add(device.getId()); + if (balancingService.isManagedByCurrentTransport(device.getId().getId())) { + establishDeviceSession(device); + } + } else { + if (balancingService.isManagedByCurrentTransport(device.getId().getId())) { + DeviceSessionContext sessionContext = sessions.get(device.getId()); + if (transportType == DeviceTransportType.SNMP) { + if (sessionContext != null) { + updateDeviceSession(sessionContext, device, deviceProfileCache.get(device.getDeviceProfileId())); + } else { + establishDeviceSession(device); + } + } else { + log.trace("Transport type was changed to {}", transportType); + destroyDeviceSession(sessionContext); + } + } + } + } + + public void onDeviceDeleted(DeviceSessionContext sessionContext) { + destroyDeviceSession(sessionContext); + } + + public void onDeviceProfileUpdated(DeviceProfile deviceProfile, DeviceSessionContext sessionContext) { + updateDeviceSession(sessionContext, sessionContext.getDevice(), deviceProfile); + } + + public void onSnmpTransportListChanged() { + log.trace("SNMP transport list changed. Updating sessions"); + List deleted = new LinkedList<>(); + for (DeviceId deviceId : allSnmpDevicesIds) { + if (balancingService.isManagedByCurrentTransport(deviceId.getId())) { + if (!sessions.containsKey(deviceId)) { + Device device = protoEntityService.getDeviceById(deviceId); + if (device != null) { + log.info("SNMP device {} is now managed by current transport node", deviceId); + establishDeviceSession(device); + } else { + deleted.add(deviceId); + } + } + } else { + Optional.ofNullable(sessions.get(deviceId)) + .ifPresent(sessionContext -> { + log.info("SNMP session for device {} is not managed by current transport node anymore", deviceId); + destroyDeviceSession(sessionContext); + }); + } + } + log.trace("Removing deleted SNMP devices: {}", deleted); + allSnmpDevicesIds.removeAll(deleted); + } + + + public Collection getSessions() { + return sessions.values(); + } + + public Map> getProfilesPdus() { + return profilesPdus; + } + + public Optional getAttributesMapping(DeviceProfileId deviceProfileId, OID responseOid) { + if (profilesTransportConfigs.containsKey(deviceProfileId)) { + return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getAttributes()); + } + return Optional.empty(); + } + + public Optional getTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) { + if (profilesTransportConfigs.containsKey(deviceProfileId)) { + return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getTelemetry()); + } + return Optional.empty(); + } + + private Optional getMapping(OID responseOid, List mappings) { + return mappings.stream() + .filter(kvMapping -> new OID(kvMapping.getOid()).equals(responseOid)) + //TODO: OID shouldn't be duplicated in the config, add backend and UI verification + .findFirst(); + } + + public ExecutorService getSnmpCallbackExecutor() { + return snmpTransportService.getSnmpCallbackExecutor(); + } + private int getSnmpMethod(String configMethod) { switch (configMethod) { case "get": diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java deleted file mode 100644 index 5d8e4d84b4..0000000000 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.transport.snmp; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.snmp4j.Snmp; -import org.snmp4j.transport.DefaultUdpTransportMapping; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.DeviceInfo; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.DeviceTransportType; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.dao.device.DeviceProfileService; -import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.transport.snmp.session.DeviceSessionCtx; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.io.IOException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -@Service("SnmpTransportService") -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.snmp.enabled}'=='true')") -@Slf4j -public class SnmpTransportService { - - private static final int ENTITY_PACK_LIMIT = 1024; - - @Autowired - private SnmpTransportContext snmpTransportContext; - - @Autowired - DeviceProfileService deviceProfileService; - - @Autowired - TenantService tenantService; - - @Autowired - DeviceService deviceService; - - @Getter - private ExecutorService snmpCallbackExecutor; - private Snmp snmp; - private ScheduledExecutorService pollingExecutor; - - @PostConstruct - public void init() { - log.info("Starting SNMP transport..."); - pollingExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("snmp-polling")); - //TODO: Set parallelism value in the config - snmpCallbackExecutor = Executors.newWorkStealingPool(20); - initializeSnmp(); - log.info("SNMP transport started!"); - } - - @PreDestroy - public void shutdown() { - log.info("Stopping SNMP transport!"); - if (pollingExecutor != null) { - pollingExecutor.shutdownNow(); - } - if (snmpCallbackExecutor != null) { - snmpCallbackExecutor.shutdownNow(); - } - if (snmp != null) { - try { - snmp.close(); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - } - log.info("SNMP transport stopped!"); - } - - @EventListener(ApplicationReadyEvent.class) - @Order(value = 2) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting SNMP polling."); - initSessionCtxList(); - startPolling(); - } - - private void initializeSnmp() { - try { - this.snmp = new Snmp(new DefaultUdpTransportMapping()); - this.snmp.listen(); - } catch (IOException e) { - //TODO: what should be done if transport wasn't initialized? - log.error(e.getMessage(), e); - } - } - - private void initSessionCtxList() { - //TODO: This approach works for monolith, in cluster the same data will be fetched by each node. - for (Tenant tenant : new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT)) { - TenantId tenantId = tenant.getTenantId(); - for (DeviceProfile deviceProfile : new PageDataIterable<>(pageLink -> deviceProfileService.findDeviceProfiles(tenantId, pageLink), ENTITY_PACK_LIMIT)) { - if (DeviceTransportType.SNMP.equals(deviceProfile.getTransportType())) { - snmpTransportContext.getProfileTransportConfig().put(deviceProfile.getId(), - (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration()); - initDeviceSessions(deviceProfile); - } - } - } - snmpTransportContext.initPduListPerProfile(); - } - - private void initDeviceSessions(DeviceProfile deviceProfile) { - for (DeviceInfo deviceInfo : new PageDataIterable<>(pageLink -> deviceService.findDeviceInfosByTenantIdAndDeviceProfileId(deviceProfile.getTenantId(), deviceProfile.getId(), pageLink), ENTITY_PACK_LIMIT)) { - snmpTransportContext.updateDeviceSessionCtx(deviceInfo, deviceProfile, snmp); - } - } - - private void startPolling() { - //TODO: Get poll period from configuration; - int poolPeriodSeconds = 1; - pollingExecutor.scheduleAtFixedRate(() -> snmpTransportContext.getDeviceSessions().values().forEach(DeviceSessionCtx::executeSnmpRequest), - 0, poolPeriodSeconds, TimeUnit.SECONDS); - } -} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/ServiceListChangedEventListener.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/ServiceListChangedEventListener.java new file mode 100644 index 0000000000..90b82abf96 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/ServiceListChangedEventListener.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.event; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.service.SnmpTransportBalancingService; + +@TbSnmpTransportComponent +@Component +@RequiredArgsConstructor +public class ServiceListChangedEventListener extends TbApplicationEventListener { + private final SnmpTransportBalancingService snmpTransportBalancingService; + + @Override + protected void onTbApplicationEvent(ServiceListChangedEvent event) { + snmpTransportBalancingService.onServiceListChanged(event); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEvent.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEvent.java new file mode 100644 index 0000000000..f7a4259b91 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEvent.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.event; + +import org.thingsboard.server.queue.discovery.event.TbApplicationEvent; + +public class SnmpTransportListChangedEvent extends TbApplicationEvent { + public SnmpTransportListChangedEvent() { + super(new Object()); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEventListener.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEventListener.java new file mode 100644 index 0000000000..1387461037 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEventListener.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.event; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.SnmpTransportContext; + +@TbSnmpTransportComponent +@Component +@RequiredArgsConstructor +public class SnmpTransportListChangedEventListener extends TbApplicationEventListener { + private final SnmpTransportContext snmpTransportContext; + + @Override + protected void onTbApplicationEvent(SnmpTransportListChangedEvent event) { + snmpTransportContext.onSnmpTransportListChanged(); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java new file mode 100644 index 0000000000..1f639daa2a --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.service; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; + +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +@TbSnmpTransportComponent +@Service +@RequiredArgsConstructor +public class ProtoTransportEntityService { + private final TransportService transportService; + private final DataDecodingEncodingService dataDecodingEncodingService; + + public Device getDeviceById(DeviceId id) { + TransportProtos.GetDeviceResponseMsg deviceProto = transportService.getDevice(TransportProtos.GetDeviceRequestMsg.newBuilder() + .setDeviceIdMSB(id.getId().getMostSignificantBits()) + .setDeviceIdLSB(id.getId().getLeastSignificantBits()) + .build()); + + if (deviceProto == null) { + return null; + } + + DeviceProfileId deviceProfileId = new DeviceProfileId(new UUID( + deviceProto.getDeviceProfileIdMSB(), deviceProto.getDeviceProfileIdLSB()) + ); + + Device device = new Device(); + device.setId(id); + device.setDeviceProfileId(deviceProfileId); + + DeviceTransportConfiguration deviceTransportConfiguration = (DeviceTransportConfiguration) dataDecodingEncodingService.decode( + deviceProto.getDeviceTransportConfiguration().toByteArray() + ).orElseThrow(() -> new IllegalStateException("Can't find device transport configuration")); + + DeviceData deviceData = new DeviceData(); + deviceData.setTransportConfiguration(deviceTransportConfiguration); + device.setDeviceData(deviceData); + + return device; + } + + public DeviceCredentials getDeviceCredentialsByDeviceId(DeviceId deviceId) { + TransportProtos.GetDeviceCredentialsResponseMsg deviceCredentialsResponse = transportService.getDeviceCredentials( + TransportProtos.GetDeviceCredentialsRequestMsg.newBuilder() + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .build() + ); + + return (DeviceCredentials) dataDecodingEncodingService.decode(deviceCredentialsResponse.getDeviceCredentialsData().toByteArray()) + .orElseThrow(() -> new IllegalArgumentException("Device credentials not found")); + } + + public List getAllSnmpDevicesIds() { + TransportProtos.GetSnmpDevicesResponseMsg devicesIdsResponse = transportService.getSnmpDevicesIds( + TransportProtos.GetSnmpDevicesRequestMsg.getDefaultInstance() + ); + + return devicesIdsResponse.getIdsList().stream() + .map(UUID::fromString) + .collect(Collectors.toList()); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportBalancingService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportBalancingService.java new file mode 100644 index 0000000000..3564ab319e --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportBalancingService.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.event.SnmpTransportListChangedEvent; + +import java.util.Comparator; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@TbSnmpTransportComponent +@Service +@RequiredArgsConstructor +@Slf4j +public class SnmpTransportBalancingService { + private final PartitionService partitionService; + private final ApplicationEventPublisher eventPublisher; + private final SnmpTransportService snmpTransportService; + + private int snmpTransportsCount = 1; + private Integer currentTransportPartitionIndex = 0; + + public void onServiceListChanged(ServiceListChangedEvent event) { + log.trace("Got service list changed event: {}", event); + recalculatePartitions(event.getOtherServices(), event.getCurrentService()); + } + + public boolean isManagedByCurrentTransport(UUID entityId) { + boolean isManaged = resolvePartitionIndexForEntity(entityId) == currentTransportPartitionIndex; + if (!isManaged) { + log.trace("Entity {} is not managed by current SNMP transport node", entityId); + } + return isManaged; + } + + private int resolvePartitionIndexForEntity(UUID entityId) { + return partitionService.resolvePartitionIndex(entityId, snmpTransportsCount); + } + + private void recalculatePartitions(List otherServices, ServiceInfo currentService) { + log.info("Recalculating partitions for SNMP transports"); + List snmpTransports = Stream.concat(otherServices.stream(), Stream.of(currentService)) + .filter(service -> service.getTransportsList().contains(snmpTransportService.getName())) + .sorted(Comparator.comparing(ServiceInfo::getServiceId)) + .collect(Collectors.toList()); + log.trace("Found SNMP transports: {}", snmpTransports); + + int previousCurrentTransportPartitionIndex = currentTransportPartitionIndex; + int previousSnmpTransportsCount = snmpTransportsCount; + + if (!snmpTransports.isEmpty()) { + for (int i = 0; i < snmpTransports.size(); i++) { + if (snmpTransports.get(i).equals(currentService)) { + currentTransportPartitionIndex = i; + break; + } + } + snmpTransportsCount = snmpTransports.size(); + } + + if (snmpTransportsCount != previousSnmpTransportsCount || currentTransportPartitionIndex != previousCurrentTransportPartitionIndex) { + log.info("SNMP transports partitions have changed: transports count = {}, current transport partition index = {}", snmpTransportsCount, currentTransportPartitionIndex); + eventPublisher.publishEvent(new SnmpTransportListChangedEvent()); + } else { + log.info("SNMP transports partitions have not changed"); + } + } + +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/SnmpSessionListener.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java similarity index 60% rename from common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/SnmpSessionListener.java rename to common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 35aa1a9347..22e60c98ee 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/SnmpSessionListener.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.snmp.session; +package org.thingsboard.server.transport.snmp.service; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -25,9 +25,16 @@ import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; import org.snmp4j.Snmp; import org.snmp4j.event.ResponseEvent; -import org.snmp4j.event.ResponseListener; import org.snmp4j.smi.VariableBinding; +import org.snmp4j.transport.DefaultUdpTransportMapping; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.transport.TransportContext; @@ -39,27 +46,116 @@ import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.snmp.SnmpTransportContext; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; +import javax.annotation.PreDestroy; +import java.io.IOException; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +@TbSnmpTransportComponent +@Service @Slf4j -@AllArgsConstructor -public class SnmpSessionListener implements ResponseListener { - - @Getter +public class SnmpTransportService implements TbTransportService { private final SnmpTransportContext snmpTransportContext; @Getter - private final String token; + private ExecutorService snmpCallbackExecutor; + @Getter + private Snmp snmp; + private ScheduledExecutorService pollingExecutor; - @Override - public void onResponse(ResponseEvent event) { - ((Snmp) event.getSource()).cancel(event.getRequest(), this); - snmpTransportContext.getSnmpCallbackExecutor().submit(() -> processSnmpResponse(event)); + public SnmpTransportService(@Lazy SnmpTransportContext snmpTransportContext) { + this.snmpTransportContext = snmpTransportContext; } - private void processSnmpResponse(ResponseEvent event) { + // @PostConstruct + private void init() { + log.info("Starting SNMP transport..."); + pollingExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("snmp-polling")); + //TODO: Set parallelism value in the config + snmpCallbackExecutor = Executors.newWorkStealingPool(20); + + initializeSnmp(); + + log.info("SNMP transport started!"); + } + + @PreDestroy + public void shutdown() { + log.info("Stopping SNMP transport!"); + if (pollingExecutor != null) { + pollingExecutor.shutdownNow(); + } + if (snmpCallbackExecutor != null) { + snmpCallbackExecutor.shutdownNow(); + } + if (snmp != null) { + try { + snmp.close(); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + } + log.info("SNMP transport stopped!"); + } + + @EventListener(ApplicationReadyEvent.class) + @Order(value = 10) + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + log.info("Received application ready event. Starting SNMP polling."); +// startPolling(); + } + + private void initializeSnmp() { + try { + this.snmp = new Snmp(new DefaultUdpTransportMapping()); + this.snmp.listen(); + } catch (IOException e) { + //TODO: what should be done if transport wasn't initialized? + log.error(e.getMessage(), e); + } + } + + private void startPolling() { + //TODO: Get poll period from configuration; + int pollPeriodSeconds = 1; + + pollingExecutor.scheduleWithFixedDelay(() -> { + snmpTransportContext.getSessions().forEach(this::executeSnmpRequest); + }, 0, pollPeriodSeconds, TimeUnit.SECONDS); + } + + private void executeSnmpRequest(DeviceSessionContext sessionContext) { + long timeNow = System.currentTimeMillis(); + long nextRequestExecutionTime = sessionContext.getPreviousRequestExecutedAt() + sessionContext.getProfileTransportConfiguration().getPollPeriodMs(); + + if (nextRequestExecutionTime < timeNow) { + sessionContext.setPreviousRequestExecutedAt(timeNow); + + DeviceProfileId deviceProfileId = sessionContext.getDeviceProfile().getId(); + snmpTransportContext.getProfilesPdus().get(deviceProfileId).forEach(pdu -> { + try { + log.debug("[{}] Sending SNMP message...", pdu.getRequestID()); + snmp.send(pdu, sessionContext.getTarget(), deviceProfileId, sessionContext); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + }); + } + } + + public void onNewDeviceResponse(ResponseEvent responseEvent, DeviceSessionContext sessionContext) { + ((Snmp) responseEvent.getSource()).cancel(responseEvent.getRequest(), sessionContext); + snmpTransportContext.getSnmpCallbackExecutor().submit(() -> processSnmpResponse(responseEvent, sessionContext)); + } + + private void processSnmpResponse(ResponseEvent event, DeviceSessionContext sessionContext) { PDU response = event.getResponse(); if (event.getError() != null) { log.warn("Response error: {}", event.getError().getMessage(), event.getError()); @@ -72,8 +168,8 @@ public class SnmpSessionListener implements ResponseListener { TransportService transportService = snmpTransportContext.getTransportService(); for (int i = 0; i < response.size(); i++) { VariableBinding vb = response.get(i); - snmpTransportContext.findAttributesMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(token).build(), + snmpTransportContext.getAttributesMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(sessionContext.getToken()).build(), new DeviceAuthCallback(snmpTransportContext, sessionInfo -> { try { transportService.process(sessionInfo, @@ -84,8 +180,8 @@ public class SnmpSessionListener implements ResponseListener { log.warn("Failed to process SNMP response: {}", e.getMessage(), e); } }))); - snmpTransportContext.findTelemetryMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(token).build(), + snmpTransportContext.getTelemetryMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(sessionContext.getToken()).build(), new DeviceAuthCallback(snmpTransportContext, sessionInfo -> { try { transportService.process(sessionInfo, @@ -103,6 +199,14 @@ public class SnmpSessionListener implements ResponseListener { } } + private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { + snmpTransportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() + .setAttributeSubscription(false) + .setRpcSubscription(false) + .setLastActivityTime(System.currentTimeMillis()) + .build(), TransportServiceCallback.EMPTY); + } + private TransportProtos.PostAttributeMsg convertToPostAttributes(String keyName, DataType dataType, String payload) throws AdaptorException { try { return JsonConverter.convertToAttributesProto(getKvJson(keyName, dataType, payload)); @@ -143,14 +247,6 @@ public class SnmpSessionListener implements ResponseListener { return new JsonParser().parse(result.toString()); } - private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { - snmpTransportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() - .setAttributeSubscription(false) - .setRpcSubscription(false) - .setLastActivityTime(System.currentTimeMillis()) - .build(), TransportServiceCallback.EMPTY); - } - @AllArgsConstructor private static class DeviceAuthCallback implements TransportServiceCallback { private final TransportContext transportContext; @@ -170,4 +266,9 @@ public class SnmpSessionListener implements ResponseListener { log.warn("Failed to process device auth", e); } } + + @Override + public String getName() { + return "SNMP"; + } } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java new file mode 100644 index 0000000000..ed87bbc726 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java @@ -0,0 +1,166 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.session; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.snmp4j.CommunityTarget; +import org.snmp4j.Target; +import org.snmp4j.event.ResponseEvent; +import org.snmp4j.event.ResponseListener; +import org.snmp4j.mp.SnmpConstants; +import org.snmp4j.smi.GenericAddress; +import org.snmp4j.smi.OctetString; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.transport.SessionMsgListener; +import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; +import org.thingsboard.server.transport.snmp.SnmpTransportContext; +import org.thingsboard.server.transport.snmp.service.SnmpTransportService; + +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class DeviceSessionContext extends DeviceAwareSessionContext implements SessionMsgListener, ResponseListener { + @Getter + private Target target; + private final String token; + @Getter + @Setter + private SnmpProfileTransportConfiguration profileTransportConfiguration; + @Getter + @Setter + private SnmpDeviceTransportConfiguration deviceTransportConfiguration; + @Getter + private final Device device; + + private final SnmpTransportContext snmpTransportContext; + private final SnmpTransportService snmpTransportService; + + @Getter + @Setter + private long previousRequestExecutedAt = 0; + private final AtomicInteger msgIdSeq = new AtomicInteger(0); + private boolean isActive = true; + + public DeviceSessionContext(Device device, DeviceProfile deviceProfile, + String token, SnmpDeviceTransportConfiguration deviceTransportConfiguration, + SnmpTransportContext snmpTransportContext, SnmpTransportService snmpTransportService) { + super(UUID.randomUUID()); + super.setDeviceId(device.getId()); + super.setDeviceProfile(deviceProfile); + this.device = device; + + this.token = token; + this.snmpTransportContext = snmpTransportContext; + this.snmpTransportService = snmpTransportService; + + this.profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + this.deviceTransportConfiguration = deviceTransportConfiguration; + + initTarget(this.profileTransportConfiguration, this.deviceTransportConfiguration); + } + + @Override + public void onDeviceProfileUpdate(TransportProtos.SessionInfoProto newSessionInfo, DeviceProfile deviceProfile) { + super.onDeviceProfileUpdate(newSessionInfo, deviceProfile); + if (isActive) { + snmpTransportContext.onDeviceProfileUpdated(deviceProfile, this); + } + } + + @Override + public void onDeviceDeleted(DeviceId deviceId) { + snmpTransportContext.onDeviceDeleted(this); + } + + @Override + public void onResponse(ResponseEvent event) { + if (isActive) { + snmpTransportService.onNewDeviceResponse(event, this); + } + } + + public void initTarget(SnmpProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { + log.trace("Initializing target for SNMP session of device {}", device); + CommunityTarget communityTarget = new CommunityTarget(); + communityTarget.setAddress(GenericAddress.parse(GenericAddress.TYPE_UDP + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort())); + communityTarget.setVersion(getSnmpVersion(deviceTransportConfig.getProtocolVersion())); + communityTarget.setCommunity(new OctetString(deviceTransportConfig.getCommunity())); + communityTarget.setTimeout(profileTransportConfig.getTimeoutMs()); + communityTarget.setRetries(profileTransportConfig.getRetries()); + this.target = communityTarget; + log.info("SNMP target initialized: {}", this.target); + } + + public void close() { + isActive = false; + } + + public String getToken() { + return token; + } + + //TODO: replace with enum, wtih preliminary discussion of type version in config (string or integer) + private int getSnmpVersion(String configSnmpVersion) { + switch (configSnmpVersion) { + case ("v1"): + return SnmpConstants.version1; + case ("v2c"): + return SnmpConstants.version2c; + case ("v3"): + return SnmpConstants.version3; + default: + return -1; + } + } + + @Override + public int nextMsgId() { + return msgIdSeq.incrementAndGet(); + } + + @Override + public void onGetAttributesResponse(GetAttributeResponseMsg getAttributesResponse) { + } + + @Override + public void onAttributeUpdate(AttributeUpdateNotificationMsg attributeUpdateNotification) { + } + + @Override + public void onRemoteSessionCloseCommand(SessionCloseNotificationProto sessionCloseNotification) { + } + + @Override + public void onToDeviceRpcRequest(ToDeviceRpcRequestMsg toDeviceRequest) { + } + + @Override + public void onToServerRpcResponse(ToServerRpcResponseMsg toServerResponse) { + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java deleted file mode 100644 index 424b964387..0000000000 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.transport.snmp.session; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.snmp4j.CommunityTarget; -import org.snmp4j.Snmp; -import org.snmp4j.Target; -import org.snmp4j.mp.SnmpConstants; -import org.snmp4j.smi.GenericAddress; -import org.snmp4j.smi.OctetString; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.DeviceTransportType; -import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.transport.SessionMsgListener; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.auth.SessionInfoCreator; -import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; -import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; -import org.thingsboard.server.transport.snmp.SnmpTransportContext; - -import java.io.IOException; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; - -@Slf4j -public class DeviceSessionCtx extends DeviceAwareSessionContext implements SessionMsgListener { - private final AtomicInteger msgIdSeq = new AtomicInteger(0); - - @Getter - @Setter - private SnmpDeviceTransportConfiguration deviceTransportConfig; - @Getter - @Setter - private SnmpSessionListener snmpSessionListener; - @Getter - @Setter - private Target target; - @Getter - @Setter - private volatile TransportProtos.SessionInfoProto sessionInfo; - - private Snmp snmp; - private SnmpProfileTransportConfiguration snmpProfileTransportConfiguration; - private long previousRequestExecutedAt = 0; - - public DeviceSessionCtx(SnmpTransportContext transportContext, String token, SnmpDeviceTransportConfiguration deviceTransportConfig, - Snmp snmp, DeviceId deviceId, DeviceProfile deviceProfile) { - super(UUID.randomUUID()); - this.snmpSessionListener = new SnmpSessionListener(transportContext, token); - super.setDeviceId(deviceId); - super.setDeviceProfile(deviceProfile); - //TODO: What should be done if snmp null? - if (snmp != null) { - this.snmp = snmp; - } - this.snmpProfileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); - initTarget(this.snmpProfileTransportConfiguration, deviceTransportConfig); - } - - @Override - public int nextMsgId() { - return msgIdSeq.incrementAndGet(); - } - - @Override - public void onGetAttributesResponse(GetAttributeResponseMsg getAttributesResponse) { - } - - @Override - public void onAttributeUpdate(AttributeUpdateNotificationMsg attributeUpdateNotification) { - } - - @Override - public void onRemoteSessionCloseCommand(SessionCloseNotificationProto sessionCloseNotification) { - } - - @Override - public void onToDeviceRpcRequest(ToDeviceRpcRequestMsg toDeviceRequest) { - } - - @Override - public void onToServerRpcResponse(ToServerRpcResponseMsg toServerResponse) { - } - - @Override - public void onDeviceProfileUpdate(TransportProtos.SessionInfoProto newSessionInfo, DeviceProfile deviceProfile) { - super.onDeviceProfileUpdate(sessionInfo, deviceProfile); - if (DeviceTransportType.SNMP.equals(deviceProfile.getTransportType())) { - snmpProfileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); - snmpSessionListener.getSnmpTransportContext().getProfileTransportConfig().put( - deviceProfile.getId(), - snmpProfileTransportConfiguration); - snmpSessionListener.getSnmpTransportContext().updatePduListPerProfile(deviceProfile.getId(), snmpProfileTransportConfiguration); - } else { - //TODO: should the context be removed from the map? - } - } - - @Override - public void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, Device device, Optional deviceProfileOpt) { - super.onDeviceUpdate(sessionInfo, device, deviceProfileOpt); - if (super.getDeviceProfile() != null && DeviceTransportType.SNMP.equals(super.getDeviceProfile().getTransportType())) { - snmpSessionListener.getSnmpTransportContext().updateDeviceSessionCtx(device, deviceProfile, null); - SnmpProfileTransportConfiguration profileTransportConfig = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); - SnmpDeviceTransportConfiguration deviceTransportConfig = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); - initTarget(profileTransportConfig, deviceTransportConfig); - } else { - //TODO: should the context be removed from the map? - } - } - - public void createSessionInfo(Consumer registerSession) { - getSnmpSessionListener().getSnmpTransportContext().getTransportService().process(DeviceTransportType.SNMP, - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(getSnmpSessionListener().getToken()).build(), - new TransportServiceCallback() { - @Override - public void onSuccess(ValidateDeviceCredentialsResponse msg) { - if (msg.hasDeviceInfo()) { - sessionInfo = SessionInfoCreator.create(msg, getSnmpSessionListener().getSnmpTransportContext(), UUID.randomUUID()); - registerSession.accept(sessionInfo); - setDeviceInfo(msg.getDeviceInfo()); - } else { - log.warn("[{}] Failed to process device auth", getDeviceId()); - } - } - - @Override - public void onError(Throwable e) { - log.warn("[{}] Failed to process device auth", getDeviceId(), e); - } - }); - } - - public void executeSnmpRequest() { - long timeNow = System.currentTimeMillis(); - long nextRequestExecutionTime = previousRequestExecutedAt + snmpProfileTransportConfiguration.getPoolPeriodMs(); - if (nextRequestExecutionTime < timeNow) { - previousRequestExecutedAt = timeNow; - - snmpSessionListener.getSnmpTransportContext().getPdusPerProfile().get(deviceProfile.getId()).forEach(pdu -> { - try { - log.debug("[{}] Sending SNMP message...", pdu.getRequestID()); - snmp.send(pdu, - target, - deviceProfile.getId(), - snmpSessionListener); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - }); - } - } - - private void initTarget(SnmpProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { - this.deviceTransportConfig = deviceTransportConfig; - CommunityTarget communityTarget = new CommunityTarget(); - communityTarget.setAddress(GenericAddress.parse(GenericAddress.TYPE_UDP + ":" + this.deviceTransportConfig.getAddress() + "/" + this.deviceTransportConfig.getPort())); - communityTarget.setVersion(getSnmpVersion(this.deviceTransportConfig.getProtocolVersion())); - communityTarget.setCommunity(new OctetString(this.deviceTransportConfig.getCommunity())); - communityTarget.setTimeout(profileTransportConfig.getTimeoutMs()); - communityTarget.setRetries(profileTransportConfig.getRetries()); - this.target = communityTarget; - log.info("SNMP target initialized: {}", this.target); - } - - //TODO: replace with enum, wtih preliminary discussion of type version in config (string or integer) - private int getSnmpVersion(String configSnmpVersion) { - switch (configSnmpVersion) { - case ("v1"): - return SnmpConstants.version1; - case ("v2c"): - return SnmpConstants.version2c; - case ("v3"): - return SnmpConstants.version3; - default: - return -1; - } - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/DeviceUpdatedEvent.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/DeviceUpdatedEvent.java new file mode 100644 index 0000000000..ad4819fbb1 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/DeviceUpdatedEvent.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport; + +import lombok.Getter; +import org.thingsboard.server.common.data.Device; + +@Getter +public class DeviceUpdatedEvent { + private final Device device; + + public DeviceUpdatedEvent(Device device) { + this.device = device; + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgListener.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgListener.java index 8209fe2531..9aff642708 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgListener.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgListener.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.transport; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; @@ -49,4 +50,7 @@ public interface SessionMsgListener { default void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, Device device, Optional deviceProfileOpt) { } + + default void onDeviceDeleted(DeviceId deviceId) { + } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java index 75f20cc825..eabcb1e8e6 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java @@ -21,6 +21,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index ab728af00d..24469dc599 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -22,11 +22,17 @@ import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsRes import org.thingsboard.server.common.transport.service.SessionMetaData; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceCredentialsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceCredentialsResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetResourcesRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetResourcesResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.LwM2MRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.LwM2MResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; @@ -55,6 +61,12 @@ public interface TransportService { GetResourcesResponseMsg getResources(GetResourcesRequestMsg msg); + GetSnmpDevicesResponseMsg getSnmpDevicesIds(GetSnmpDevicesRequestMsg requestMsg); + + GetDeviceResponseMsg getDevice(GetDeviceRequestMsg requestMsg); + + GetDeviceCredentialsResponseMsg getDeviceCredentials(GetDeviceCredentialsRequestMsg requestMsg); + void process(DeviceTransportType transportType, ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java index 5c36e9b3e0..ad289a7e16 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java @@ -112,8 +112,8 @@ public class DefaultTransportDeviceProfileCache implements TransportDeviceProfil profile = profileOpt.get(); this.put(profile); } else { - log.warn("[{}] Can't device profile: {}", id, entityProfileMsg.getData()); - throw new RuntimeException("Can't device profile!"); + log.warn("[{}] Can't find device profile: {}", id, entityProfileMsg.getData()); + throw new RuntimeException("Can't find device profile!"); } } finally { deviceProfileFetchLock.unlock(); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index b273ee48cf..903de94020 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -23,6 +23,7 @@ import com.google.gson.JsonObject; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.ApiUsageRecordKey; @@ -47,6 +48,7 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.stats.MessagesStats; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.common.transport.DeviceUpdatedEvent; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportDeviceProfileCache; import org.thingsboard.server.common.transport.TransportService; @@ -61,7 +63,6 @@ import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -131,6 +132,7 @@ public class DefaultTransportService implements TransportService { private final TransportRateLimitService rateLimitService; private final DataDecodingEncodingService dataDecodingEncodingService; private final SchedulerComponent scheduler; + private final ApplicationEventPublisher eventPublisher; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; protected TbQueueProducer> ruleEngineMsgProducer; @@ -157,7 +159,8 @@ public class DefaultTransportService implements TransportService { TransportDeviceProfileCache deviceProfileCache, TransportTenantProfileCache tenantProfileCache, TbApiUsageClient apiUsageClient, TransportRateLimitService rateLimitService, - DataDecodingEncodingService dataDecodingEncodingService, SchedulerComponent scheduler) { + DataDecodingEncodingService dataDecodingEncodingService, SchedulerComponent scheduler, + ApplicationEventPublisher eventPublisher) { this.serviceInfoProvider = serviceInfoProvider; this.queueProvider = queueProvider; this.producerProvider = producerProvider; @@ -169,6 +172,7 @@ public class DefaultTransportService implements TransportService { this.rateLimitService = rateLimitService; this.dataDecodingEncodingService = dataDecodingEncodingService; this.scheduler = scheduler; + this.eventPublisher = eventPublisher; } @PostConstruct @@ -266,6 +270,58 @@ public class DefaultTransportService implements TransportService { } } + @Override + public TransportProtos.GetSnmpDevicesResponseMsg getSnmpDevicesIds(TransportProtos.GetSnmpDevicesRequestMsg requestMsg) { + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>( + UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder() + .setSnmpDevicesRequestMsg(requestMsg) + .build() + ); + + try { + TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); + return response.getValue().getSnmpDevicesResponseMsg(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + @Override + public TransportProtos.GetDeviceResponseMsg getDevice(TransportProtos.GetDeviceRequestMsg requestMsg) { + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>( + UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder() + .setDeviceRequestMsg(requestMsg) + .build() + ); + + try { + TransportApiResponseMsg response = transportApiRequestTemplate.send(protoMsg).get().getValue(); + if (response.hasDeviceResponseMsg()) { + return response.getDeviceResponseMsg(); + } else { + return null; + } + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + @Override + public TransportProtos.GetDeviceCredentialsResponseMsg getDeviceCredentials(TransportProtos.GetDeviceCredentialsRequestMsg requestMsg) { + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>( + UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder() + .setDeviceCredentialsRequestMsg(requestMsg) + .build() + ); + + try { + TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); + return response.getValue().getDeviceCredentialsResponseMsg(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + @Override public void process(DeviceTransportType transportType, TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { @@ -685,7 +741,10 @@ public class DefaultTransportService implements TransportService { } } else if (EntityType.DEVICE.equals(entityType)) { Optional deviceOpt = dataDecodingEncodingService.decode(msg.getData().toByteArray()); - deviceOpt.ifPresent(this::onDeviceUpdate); + deviceOpt.ifPresent(device -> { + onDeviceUpdate(device); + eventPublisher.publishEvent(new DeviceUpdatedEvent(device)); + }); } } else if (toSessionMsg.hasEntityDeleteMsg()) { TransportProtos.EntityDeleteMsg msg = toSessionMsg.getEntityDeleteMsg(); @@ -699,6 +758,7 @@ public class DefaultTransportService implements TransportService { rateLimitService.remove(new TenantId(entityUuid)); } else if (EntityType.DEVICE.equals(entityType)) { rateLimitService.remove(new DeviceId(entityUuid)); + onDeviceDeleted(new DeviceId(entityUuid)); } } else if (toSessionMsg.hasResourceUpdateMsg()) { //TODO: update resource cache @@ -764,6 +824,17 @@ public class DefaultTransportService implements TransportService { }); } + private void onDeviceDeleted(DeviceId deviceId) { + sessions.forEach((id, md) -> { + DeviceId sessionDeviceId = new DeviceId(new UUID(md.getSessionInfo().getDeviceIdMSB(), md.getSessionInfo().getDeviceIdLSB())); + if (sessionDeviceId.equals(deviceId)) { + transportCallbackExecutor.submit(() -> { + md.getListener().onDeviceDeleted(deviceId); + }); + } + }); + } + protected UUID toSessionId(TransportProtos.SessionInfoProto sessionInfo) { return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 7f8d512c70..547725b635 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.device; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -215,4 +216,7 @@ public interface DeviceDao extends Dao, TenantEntityDao { * @return the list of device objects */ PageData findDevicesByTenantIdAndProfileId(UUID tenantId, UUID profileId, PageLink pageLink); + + List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index f933cbd543..64b44db686 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -80,6 +81,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -554,6 +556,11 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe return savedDevice; } + @Override + public List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType) { + return deviceDao.findDevicesIdsByDeviceProfileTransportType(transportType); + } + private DataValidator deviceValidator = new DataValidator() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index c39e481638..708e0f38e3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; @@ -170,4 +171,9 @@ public interface DeviceRepository extends PagingAndSortingRepository findIdsByDeviceProfileTransportType(@Param("transportType") DeviceTransportType transportType); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 18979eba91..cb91ffac4f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; @@ -114,6 +115,11 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao DaoUtil.toPageable(pageLink))); } + @Override + public List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType) { + return deviceRepository.findIdsByDeviceProfileTransportType(transportType); + } + @Override public PageData findDeviceInfosByTenantIdAndCustomerId(UUID tenantId, UUID customerId, PageLink pageLink) { return DaoUtil.toPageData( diff --git a/docker/tb-transports/snmp/conf/logback.xml b/docker/tb-transports/snmp/conf/logback.xml new file mode 100644 index 0000000000..e581a97c42 --- /dev/null +++ b/docker/tb-transports/snmp/conf/logback.xml @@ -0,0 +1,50 @@ + + + + + + + /var/log/tb-snmp-transport/${TB_SERVICE_ID}/tb-snmp-transport.log + + /var/log/tb-snmp-transport/${TB_SERVICE_ID}/tb-snmp-transport.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + \ No newline at end of file diff --git a/docker/tb-transports/snmp/conf/tb-snmp-transport.conf b/docker/tb-transports/snmp/conf/tb-snmp-transport.conf new file mode 100644 index 0000000000..2aceae797c --- /dev/null +++ b/docker/tb-transports/snmp/conf/tb-snmp-transport.conf @@ -0,0 +1,23 @@ +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/tb-snmp-transport/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-snmp-transport/${TB_SERVICE_ID}-heapdump.bin" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" +export LOG_FILENAME=tb-snmp-transport.out +export LOADER_PATH=/usr/share/tb-snmp-transport/conf diff --git a/msa/transport/snmp/docker/Dockerfile b/msa/transport/snmp/docker/Dockerfile index e39624d538..b808932482 100644 --- a/msa/transport/snmp/docker/Dockerfile +++ b/msa/transport/snmp/docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM thingsboard/openjdk8 +FROM thingsboard/openjdk11 COPY start-tb-snmp-transport.sh ${pkg.name}.deb /tmp/ diff --git a/transport/snmp/src/main/conf/logback.xml b/transport/snmp/src/main/conf/logback.xml new file mode 100644 index 0000000000..d9818c2257 --- /dev/null +++ b/transport/snmp/src/main/conf/logback.xml @@ -0,0 +1,45 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + diff --git a/transport/snmp/src/main/conf/tb-snmp-transport.conf b/transport/snmp/src/main/conf/tb-snmp-transport.conf new file mode 100644 index 0000000000..73b6e04f68 --- /dev/null +++ b/transport/snmp/src/main/conf/tb-snmp-transport.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/transport/snmp/src/main/resources/logback.xml b/transport/snmp/src/main/resources/logback.xml new file mode 100644 index 0000000000..7a3d9f6f0d --- /dev/null +++ b/transport/snmp/src/main/resources/logback.xml @@ -0,0 +1,36 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + \ No newline at end of file diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml new file mode 100644 index 0000000000..9f6bd131ac --- /dev/null +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -0,0 +1,394 @@ +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +server: + # Server bind address + address: "${HTTP_BIND_ADDRESS:0.0.0.0}" + # Server bind port + port: "${HTTP_BIND_PORT:8085}" + # Server SSL configuration + +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:false}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" + +cluster: + stats: + enabled: "${TB_CLUSTER_STATS_ENABLED:false}" + print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" + +cache: + # caffeine or redis + type: "${CACHE_TYPE:caffeine}" + attributes: + # make sure that if cache.type is 'redis' and cache.attributes.enabled is 'true' that you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random' + enabled: "${CACHE_ATTRIBUTES_ENABLED:true}" + +caffeine: + specs: + relations: + timeToLiveInMinutes: 1440 + maxSize: 0 + deviceCredentials: + timeToLiveInMinutes: 1440 + maxSize: 0 + devices: + timeToLiveInMinutes: 1440 + maxSize: 0 + sessions: + timeToLiveInMinutes: 1440 + maxSize: 0 + assets: + timeToLiveInMinutes: 1440 + maxSize: 0 + entityViews: + timeToLiveInMinutes: 1440 + maxSize: 0 + claimDevices: + timeToLiveInMinutes: 1 + maxSize: 0 + securitySettings: + timeToLiveInMinutes: 1440 + maxSize: 0 + tenantProfiles: + timeToLiveInMinutes: 1440 + maxSize: 0 + deviceProfiles: + timeToLiveInMinutes: 1440 + maxSize: 0 + attributes: + timeToLiveInMinutes: 1440 + maxSize: 100000 + +redis: + # standalone or cluster + connection: + type: "${REDIS_CONNECTION_TYPE:standalone}" + standalone: + host: "${REDIS_HOST:localhost}" + port: "${REDIS_PORT:6379}" + useDefaultClientConfig: "${REDIS_USE_DEFAULT_CLIENT_CONFIG:true}" + # this value may be used only if you used not default ClientConfig + clientName: "${REDIS_CLIENT_NAME:standalone}" + # this value may be used only if you used not default ClientConfig + connectTimeout: "${REDIS_CLIENT_CONNECT_TIMEOUT:30000}" + # this value may be used only if you used not default ClientConfig + readTimeout: "${REDIS_CLIENT_READ_TIMEOUT:60000}" + # this value may be used only if you used not default ClientConfig + usePoolConfig: "${REDIS_CLIENT_USE_POOL_CONFIG:false}" + cluster: + # Comma-separated list of "host:port" pairs to bootstrap from. + nodes: "${REDIS_NODES:}" + # Maximum number of redirects to follow when executing commands across the cluster. + max-redirects: "${REDIS_MAX_REDIRECTS:12}" + useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" + # db index + db: "${REDIS_DB:0}" + # db password + password: "${REDIS_PASSWORD:}" + # pool config + pool_config: + maxTotal: "${REDIS_POOL_CONFIG_MAX_TOTAL:128}" + maxIdle: "${REDIS_POOL_CONFIG_MAX_IDLE:128}" + minIdle: "${REDIS_POOL_CONFIG_MIN_IDLE:16}" + testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:true}" + testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:true}" + testWhileIdle: "${REDIS_POOL_CONFIG_TEST_WHILE_IDLE:true}" + minEvictableMs: "${REDIS_POOL_CONFIG_MIN_EVICTABLE_MS:60000}" + evictionRunsMs: "${REDIS_POOL_CONFIG_EVICTION_RUNS_MS:30000}" + maxWaitMills: "${REDIS_POOL_CONFIG_MAX_WAIT_MS:60000}" + numberTestsPerEvictionRun: "${REDIS_POOL_CONFIG_NUMBER_TESTS_PER_EVICTION_RUN:3}" + blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" + +# Check new version updates parameters +updates: + # Enable/disable updates checking. + enabled: "${UPDATES_ENABLED:true}" + +# spring freemarker configuration +spring.freemarker.checkTemplateLocation: "false" + +audit-log: + # Enable/disable audit log functionality. + enabled: "${AUDIT_LOG_ENABLED:true}" + # Specify partitioning size for audit log by tenant id storage. Example MINUTES, HOURS, DAYS, MONTHS + by_tenant_partitioning: "${AUDIT_LOG_BY_TENANT_PARTITIONING:MONTHS}" + # Number of days as history period if startTime and endTime are not specified + default_query_period: "${AUDIT_LOG_DEFAULT_QUERY_PERIOD:30}" + # Logging levels per each entity type. + # Allowed values: OFF (disable), W (log write operations), RW (log read and write operations) + logging-level: + mask: + "device": "${AUDIT_LOG_MASK_DEVICE:W}" + "asset": "${AUDIT_LOG_MASK_ASSET:W}" + "dashboard": "${AUDIT_LOG_MASK_DASHBOARD:W}" + "customer": "${AUDIT_LOG_MASK_CUSTOMER:W}" + "user": "${AUDIT_LOG_MASK_USER:W}" + "rule_chain": "${AUDIT_LOG_MASK_RULE_CHAIN:W}" + "alarm": "${AUDIT_LOG_MASK_ALARM:W}" + "entity_view": "${AUDIT_LOG_MASK_ENTITY_VIEW:W}" + "device_profile": "${AUDIT_LOG_MASK_DEVICE_PROFILE:W}" + sink: + # Type of external sink. possible options: none, elasticsearch + type: "${AUDIT_LOG_SINK_TYPE:none}" + # Name of the index where audit logs stored + # Index name could contain next placeholders (not mandatory): + # @{TENANT} - substituted by tenant ID + # @{DATE} - substituted by current date in format provided in audit_log.sink.date_format + index_pattern: "${AUDIT_LOG_SINK_INDEX_PATTERN:@{TENANT}_AUDIT_LOG_@{DATE}}" + # Date format. Details of the pattern could be found here: + # https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html + date_format: "${AUDIT_LOG_SINK_DATE_FORMAT:YYYY.MM.DD}" + scheme_name: "${AUDIT_LOG_SINK_SCHEME_NAME:http}" # http or https + host: "${AUDIT_LOG_SINK_HOST:localhost}" + port: "${AUDIT_LOG_SINK_PORT:9200}" + user_name: "${AUDIT_LOG_SINK_USER_NAME:}" + password: "${AUDIT_LOG_SINK_PASSWORD:}" + +state: + # Should be greater then transport.sessions.report_timeout + defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}" + defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}" + persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" + +transport: + sessions: + inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" + report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" + json: + # Cast String data types to Numeric if possible when processing Telemetry/Attributes JSON + type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" + # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) + max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" + # Enable/disable http/mqtt/coap transport protocols (has higher priority than certain protocol's 'enabled' property) + api_enabled: "${TB_TRANSPORT_API_ENABLED:true}" + # Local LwM2M transport parameters + snmp: + enabled: "${SNMP_ENABLED:true}" + +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + in_memory: + stats: + # For debug lvl + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + other: + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" + consumer-stats: + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" + aws_sqs: + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + transport_api: + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_eval_requests_timeout: "${REMOTE_JS_MAX_EVAL_REQUEST_TIMEOUT:60000}" + # JS max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:2000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}" + queues: + - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:2000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + max-pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:3}"# Max allowed time in seconds for pause between retries. + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:2000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + max-pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}"# Max allowed time in seconds for pause between retries. + - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" + topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" + poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:2000}" + submit-strategy: + type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}"# Max allowed time in seconds for pause between retries. + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + +event: + debug: + max-symbols: "${TB_MAX_DEBUG_EVENT_SYMBOLS:4096}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. + +metrics: + # Enable/disable actuator metrics. + enabled: "${METRICS_ENABLED:false}" + timer: + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + +management: + endpoints: + web: + exposure: + # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). + include: '${METRICS_ENDPOINTS_EXPOSE:info}'